In [3]:
import random
import time
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, ConfusionMatrixDisplay, precision_score, f1_score, recall_score
import torch
from sklearn.model_selection import train_test_split
from torch import nn, optim
from torch.utils.data import TensorDataset, DataLoader, Dataset
from sklearn.utils import compute_class_weight
from tqdm.notebook import tqdm
import matplotlib.pyplot as plt
from torch.nn.utils.rnn import pad_sequence,  pack_padded_sequence

# Set seeds
seed = 69
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)

# Set GPU
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"The device is set to: {device}")
The device is set to: cuda

Preprocessing¶

In [2]:
data_1 = pd.read_json("/kaggle/input/lab22222/train.json").drop(columns='md5hash')
data_2 = pd.read_json("/kaggle/input/lab22222/test.json").drop(columns='md5hash')

previous_length = len(data_1)
data_1 = data_1.drop_duplicates(subset=["api_call_sequence"])
print(f"Removed {previous_length - len(data_1)} duplicates \nDataset 1 length: {len(data_1)}")

previous_length = len(data_2)
data_2 = data_2.drop_duplicates(subset=["api_call_sequence"])
print(f"Removed {previous_length - len(data_2)} duplicates \nDataset 2 length: {len(data_2)}")
Removed 17875 duplicates 
Dataset 1 length: 12838
Removed 5814 duplicates 
Dataset 2 length: 7349
In [3]:
X = data_1.iloc[:, :-1].values  # Remove label column
X = [x[0] for x in X]
y = data_1.iloc[:, -1].values

X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=seed)  # 80% of the data is used for training
X_test = data_2.iloc[:, :-1].values  # Remove label column
X_test = [x[0] for x in X_test]
y_test = data_2.iloc[:, -1].values

Task 1: BoW Approaches¶

Applying vectorizer¶

In [4]:
X_train_str = [str(x) for x in X_train]
X_val_str = [str(x) for x in X_val]
X_test_str = [str(x) for x in X_test]

vectorizer = CountVectorizer()

# Computing the matrix representing the words contained in each api sequence
document_term_matrix_training = vectorizer.fit_transform(X_train_str)  # The vocabulary is computed on the training set and is used to compute the matrix for the other datasets
document_term_matrix_validation = vectorizer.transform(X_val_str)
document_term_matrix_testing = vectorizer.transform(X_test_str)

# There are 249 columns representing the vocabulary size computed on the training set
print(document_term_matrix_training.shape)
(10270, 249)

Test set out-of-vocabulary words¶

In [5]:
# Computing the words contained in the training set vocabulary which are not inside the testing set vocabulary
# The out-of-vocabulary words are ignored by CountVectorizer by default

vectorizer_testing = CountVectorizer()
vectorizer_testing.fit(X_test_str)
out_of_vocabulary = vectorizer.vocabulary_.keys() - vectorizer_testing.vocabulary_.keys()

print(f"There are {len(out_of_vocabulary)} Out-of-vocabulary words from the test set:\n{out_of_vocabulary}")
There are 26 Out-of-vocabulary words from the test set:
{'netusergetlocalgroups', 'sendto', 'ntloadkey', 'ntwritevirtualmemory', 'httpqueryinfoa', 'findfirstfileexa', 'internetqueryoptiona', 'getkeyboardstate', 'listen', 'internetreadfile', 'setinformationjobobject', 'getaddrinfow', 'sendnotifymessagew', 'ntreadvirtualmemory', 'ntqueryfullattributesfile', 'internetopenurlw', 'internetsetstatuscallback', 'setfileinformationbyhandle', 'netusergetinfo', 'ntterminatethread', 'wsasocketw', 'certopensystemstorew', 'cryptdecodeobjectex', 'getusernameexw', 'shutdown', 'certopenstore'}

Fit a classifier¶

Logistic regression¶

In [6]:
# The solver is 'saga' because it is fast for large dataset
# The max_iter is 2400 because if it is lower the solver fails to converge (it hasn't reached the global optimum* to within the specified tolerance)
# The class_weight is used to consider the different number of sample for each class (malware and benign)

classifier = LogisticRegression(random_state=seed, solver='saga', max_iter=2400, class_weight='balanced')
classifier.fit(document_term_matrix_training, y_train)

train_predictions = classifier.predict(document_term_matrix_training)
validation_predictions = classifier.predict(document_term_matrix_validation)
test_predictions = classifier.predict(document_term_matrix_testing)

# Evaluating the classifier
train_accuracy = accuracy_score(y_train, train_predictions)
validation_accuracy = accuracy_score(y_val, validation_predictions)
test_accuracy = accuracy_score(y_test, test_predictions)

train_precision = precision_score(y_train, train_predictions, average=None)
validation_precision = precision_score(y_val, validation_predictions, average=None)
test_precision = precision_score(y_test, test_predictions, average=None)

train_recall = recall_score(y_train, train_predictions, average=None)
validation_recall = recall_score(y_val, validation_predictions, average=None)
test_recall = recall_score(y_test, test_predictions, average=None)

train_f1 = f1_score(y_train, train_predictions, average=None)
validation_f1 = f1_score(y_val, validation_predictions, average=None)
test_f1 = f1_score(y_test, test_predictions, average=None)

print(f'Train Accuracy: {train_accuracy:.4f}')
print(f'Train Precision Scores: {train_precision}')
print(f'Train Recall Scores: {train_recall}')
print(f'Train F1 Scores: {train_f1}')

print(f'Validation Accuracy: {validation_accuracy:.4f}')
print(f'Validation Precision Scores: {validation_precision}')
print(f'Validation Recall Scores: {validation_recall}')
print(f'Validation F1 Scores: {validation_f1}')

print(f'Test Accuracy: {test_accuracy:.4f}')
print(f'Test Precision Scores: {test_precision}')
print(f'Test Recall Scores: {test_recall}')
print(f'Test F1 Scores: {test_f1}')

# Generate confusion matrix
ConfusionMatrixDisplay.from_predictions(y_test, test_predictions, cmap="Blues", normalize="true")
Train Accuracy: 0.8665
Train Precision Scores: [0.2281918  0.99590116]
Train Recall Scores: [0.91860465 0.86422764]
Train F1 Scores: [0.36557149 0.92540399]
Validation Accuracy: 0.8520
Validation Precision Scores: [0.17312073 0.99201503]
Validation Recall Scores: [0.8172043  0.85333333]
Validation F1 Scores: [0.28571429 0.91746308]
Test Accuracy: 0.8611
Test Precision Scores: [0.17523168 0.99318403]
Test Recall Scores: [0.832      0.86209325]
Test F1 Scores: [0.289492   0.92300731]
Out[6]:
<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x79a69ecbac10>
No description has been provided for this image

Task 2: FFNN¶

In [7]:
# Computing mean for each API sequence length for training and testing set
api_calls_training_len = [len(seq) for seq in X_train]
api_calls_testing_len = [len(seq) for seq in X_test]

print(f"Training set sequences length -> Mean: {round(np.mean(api_calls_training_len),2)} - Standard deviation: {round(np.std(api_calls_training_len), 2)}")
print(f"Testing set sequences length  -> Mean: {round(np.mean(api_calls_testing_len), 2)} - Standard deviation: {round(np.std(api_calls_testing_len), 2)}")
Training set sequences length -> Mean: 76.13 - Standard deviation: 8.86
Testing set sequences length  -> Mean: 85.75 - Standard deviation: 8.88

Using sequential IDs¶

In [8]:
# Assigning sequential IDs to training set
for i in range(0, len(X_train)):
    row = X_train[i]
    for j in range(len(row)):
        row[j] = vectorizer.vocabulary_[row[j].lower()]
    if len(row) < 100:  ## To pad the sequences with less than 100 elements
        for k in range(0, 100 - len(row)):
            row.append(max(vectorizer.vocabulary_.values()) + 1)
    X_train[i] = row

# Assigning sequential IDs to validation set
for i in range(0, len(X_val)):
    row = X_val[i]
    for j in range(len(row)):
        if row[j].lower() not in vectorizer.vocabulary_.keys():  # To manage the out-of-vocabulary words
            row[j] = max(vectorizer.vocabulary_.values()) + 2
        else:
            row[j] = vectorizer.vocabulary_[row[j].lower()]
    if len(row) < 100:  ## To pad the sequences with less than 100 elements
        for k in range(0, 100 - len(row)):
            row.append(max(vectorizer.vocabulary_.values()) + 1)
    X_val[i] = row

# Assigning sequential IDs to testing set
for i in range(0, len(X_test)):
    row = X_test[i]
    for j in range(len(row)):
        if row[j].lower() not in vectorizer.vocabulary_.keys():  # To manage the out-of-vocabulary words
            row[j] = max(vectorizer.vocabulary_.values()) + 2
        else:
            row[j] = vectorizer.vocabulary_[row[j].lower()]
    if len(row) < 100:  ## To pad the sequences with less than 100 elements
        for k in range(0, 100 - len(row)):
            row.append(max(vectorizer.vocabulary_.values()) + 1)
    X_test[i] = row
In [9]:
def training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs, patience=10, min_delta=0.001):
    start_time = time.time()
    train_losses = []
    val_losses = []

    best_val_loss = float('inf')
    epochs_no_improvement = 0

    for epoch in tqdm(range(num_epochs), desc="Training"):  # Train for epochs
        train_loss = 0
        val_loss = 0
        model.train()  # Set model to training mode
        for batch_X, batch_y in train_loader:
            batch_X, batch_y = batch_X.to(device), batch_y.to(device)  # Move batch to GPU
            optimizer.zero_grad()  # Clear previous gradients
            outputs = model(batch_X)  # Forward pass
            loss = criterion(outputs, batch_y)  # Compute loss
            loss.backward()  # Backpropagation
            optimizer.step()  # Update model parameters
            train_loss += loss.item() * batch_X.size(0)
        train_loss /= len(train_dataset)
        train_losses.append(train_loss)  # Store training loss

        # Compute Validation loss during training
        model.eval()  # Set model to evaluation mode
        with torch.no_grad():  # Disable gradient computation for validation
            for batch_X, batch_y in val_loader:
                batch_X, batch_y = batch_X.to(device), batch_y.to(device)  # Move batch to GPU
                val_outputs = model(batch_X)  # Forward pass
                loss = criterion(val_outputs, batch_y)  # Compute validation loss
                val_loss += loss.item() * batch_X.size(0)
            val_loss /= len(val_dataset)
            val_losses.append(val_loss)  # Store validation loss

        # Early stopping
        if best_val_loss - val_loss > min_delta:
            best_val_loss = val_loss
            epochs_no_improvement = 0
            torch.save(model.state_dict(), 'best_model.pth')  # Saving the best model
        else:
            epochs_no_improvement += 1
            if epochs_no_improvement >= patience:
                print(f'Early stopping triggered after {epoch + 1} epochs because there was no improvement over {patience} epochs!')
                print(f'Epoch {epoch+1}/{num_epochs}, Train Loss: {train_losses[-1]:.4f}, Val Loss: {val_losses[-1]:.4f}')
                break  # Stop training

        if (epoch + 1) % 20 == 0:
            print(f'Epoch {epoch+1}/{num_epochs}, Train Loss: {train_losses[-1]:.4f}, Val Loss: {val_losses[-1]:.4f}')

    end_time = time.time()
    elapsed_time = end_time - start_time

    print(f'The function took {elapsed_time:.2f} seconds to execute.')

    # Restore best model weights
    model.load_state_dict(torch.load('best_model.pth', map_location=device))

    # Plot the training and validation loss
    plt.figure(figsize=(10, 5))
    plt.plot(train_losses, label='Train Loss')
    plt.plot(val_losses, label='Validation Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.title('Training and Validation Loss')
    plt.legend()
    plt.grid()
    plt.show()

    return

def testing_model(model, dataloader, device):
    model.eval()  # Set the model to evaluation mode
    all_labels = []
    all_predictions = []

    with torch.no_grad():  # Disable gradient computation for efficiency
        for inputs, labels in dataloader:
            inputs, labels = inputs.to(device), labels.to(device)  # Move batch to GPU
            outputs = model(inputs)
            _, predicted = torch.max(outputs, 1)  # Get the class with the highest score
            all_labels.extend(labels.cpu().numpy())
            all_predictions.extend(predicted.cpu().numpy())

    accuracy = accuracy_score(all_labels, all_predictions) * 100
    precision = precision_score(all_labels, all_predictions, average=None) * 100
    recall = recall_score(all_labels, all_predictions, average=None) * 100
    f1 = f1_score(all_labels, all_predictions, average=None) * 100

    return accuracy, precision, recall, f1, all_labels, all_predictions

def stats(accuracy, precision, recall, f1, description):
    print(f'{description} Accuracy: {accuracy:.4f}')
    print(f'{description} Precision Scores: {precision}')
    print(f'{description} Recall Scores: {recall}')
    print(f'{description} F1 Scores: {f1}\n')
In [10]:
class NonLinearModel(nn.Module):
    def __init__(self, in_size, hid1_size, hid2_size, out_size):
        super(NonLinearModel, self).__init__()
        self.l1 = nn.Linear(in_size, hid1_size)
        self.l2 = nn.Linear(hid1_size, hid2_size)
        self.l3 = nn.Linear(hid2_size, out_size)
        self.relu = nn.ReLU()
        self.dropout = nn.Dropout(0.4)
        self.bn1 = nn.BatchNorm1d(hid1_size)
        self.bn2 = nn.BatchNorm1d(hid2_size)

    def forward(self, x):
        x = self.l1(x)
        x = self.relu(x)
        x = self.bn1(x)
        x = self.dropout(x)
        x = self.l2(x)
        x = self.bn2(x)
        x = self.relu(x)
        x = self.dropout(x)
        x = self.l3(x)
        return x
In [11]:
X_train_tensor = torch.tensor(X_train, dtype=torch.float32)
y_train_tensor = torch.tensor(y_train, dtype=torch.long)
X_val_tensor = torch.tensor(X_val, dtype=torch.float32)
y_val_tensor = torch.tensor(y_val, dtype=torch.long)
X_test_tensor = torch.tensor(X_test, dtype=torch.float32)
y_test_tensor = torch.tensor(y_test, dtype=torch.long)

train_dataset = TensorDataset(X_train_tensor, y_train_tensor)
val_dataset = TensorDataset(X_val_tensor, y_val_tensor)
test_dataset = TensorDataset(X_test_tensor, y_test_tensor)

train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=64, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=64, shuffle=False)

vocabulary_size = len(vectorizer.vocabulary_) + 2  # Add 2 for padding and for out-of-vocabulary
input_size = 100
hidden1_size = 32
hidden2_size = 16
output_size = 2

model = NonLinearModel(input_size, hidden1_size, hidden2_size, 2)
model = model.to(device)

class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)  # Computing class weights
class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)

optimizer = optim.AdamW(model.parameters(), lr=0.0001)
num_epochs = 100
training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs)

train_accuracy, train_precisionscores, train_recallscores, train_f1scores, _, _ = testing_model(model,train_loader,device)
val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model(model,val_loader,device)
test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model(model,test_loader,device)

stats(train_accuracy, train_precisionscores, train_recallscores, train_f1scores, 'Training')
stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')

ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5752, Val Loss: 0.5488
Epoch 40/100, Train Loss: 0.5371, Val Loss: 0.4975
Early stopping triggered after 58 epochs because there was no improvement over 10 epochs!
Epoch 58/100, Train Loss: 0.5186, Val Loss: 0.4879
The function took 23.14 seconds to execute.
No description has been provided for this image
Training Accuracy: 75.4138
Training Precision Scores: [11.72816953 98.55303332]
Training Recall Scores: [74.65116279 75.44715447]
Training F1 Scores: [20.27155036 85.46595292]

Validation Accuracy: 74.9221
Validation Precision Scores: [10.01451379 98.72272485]
Validation Recall Scores: [74.19354839 74.94949495]
Validation F1 Scores: [17.64705882 85.20900322]

Test Accuracy: 75.8334
Test Precision Scores: [ 9.37167199 98.64741364]
Test Recall Scores: [70.4        76.02479222]
Test F1 Scores: [16.54135338 85.87112172]

Out[11]:
<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x79a6ca210a90>
No description has been provided for this image

Using embeddings¶

In [12]:
class EmbeddingsNonLinearModel(nn.Module):
    def __init__(self, vocab_size, seq_size, embed_size, hid_size, out_size):
        super(EmbeddingsNonLinearModel, self).__init__()

        # 1) embedding layer: vocab_size × d
        self.embedding = nn.Embedding(vocab_size, embed_size, padding_idx=max(vectorizer.vocabulary_.values()) + 1)

        # 2) first linear layer:
        self.fc1 = nn.Linear(seq_size * embed_size, hid_size) # (seq_len·d) → H
        self.bn1 = nn.BatchNorm1d(hid_size)

        # 3) second linear layer: H → 2
        self.fc2 = nn.Linear(hid_size, out_size)
        self.relu = nn.ReLU()
        self.dropout = nn.Dropout(0.5)


    def forward(self, x: torch.LongTensor):
        # a) embedding lookup → (B, seq_len, d) certo, perchè se prima avevo per ogni parola d dim, se qui ho 100 parole avrò 100xd dimensioni
        x = self.embedding(x)

        # b) flatten →
        x = x.view(x.size(0), -1) # (B, seq_len·d) e infatti appatta con primo layer lineare definito sopra, in quanto quello è seq_len * d

        # c) linear + ReLU → (B, H)
        x = self.relu(self.fc1(x))
        x = self.dropout(x)
        x = self.bn1(x)

        # d) final linear layer → (B, 2)
        x = self.fc2(x)
        return x
In [13]:
X_train_tensor = torch.tensor(X_train, dtype=torch.long)
y_train_tensor = torch.tensor(y_train, dtype=torch.long)
X_val_tensor = torch.tensor(X_val, dtype=torch.long)
y_val_tensor = torch.tensor(y_val, dtype=torch.long)
X_test_tensor = torch.tensor(X_test, dtype=torch.long)
y_test_tensor = torch.tensor(y_test, dtype=torch.long)

train_dataset = TensorDataset(X_train_tensor, y_train_tensor)
val_dataset = TensorDataset(X_val_tensor, y_val_tensor)
test_dataset = TensorDataset(X_test_tensor, y_test_tensor)

train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=64, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=64, shuffle=False)

vocabulary_size_embedding = len(vectorizer.vocabulary_) + 2  # Add 2 for padding for out-of-vocabulary and padding
sequence_size = 100
embedding_size = 8
hidden_size = 32
output_size_embedding = 2

model = EmbeddingsNonLinearModel(vocabulary_size_embedding, sequence_size, embedding_size, hidden_size, output_size_embedding)
model = model.to(device)

class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)  # Computing class weights
class_weight_tensor_embedding = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
criterion = nn.CrossEntropyLoss(class_weight_tensor_embedding)

optimizer = optim.AdamW(model.parameters(), lr=0.0005, weight_decay=0.0001)
num_epochs = 100
training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs)

train_accuracy, train_precisionscores, train_recallscores, train_f1scores, _, _ = testing_model(model,train_loader,device)
val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model(model,val_loader,device)
test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model(model,test_loader,device)

stats(train_accuracy, train_precisionscores, train_recallscores, train_f1scores, 'Training')
stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')

ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 17 epochs because there was no improvement over 10 epochs!
Epoch 17/100, Train Loss: 0.1611, Val Loss: 0.3958
The function took 6.53 seconds to execute.
No description has been provided for this image
Training Accuracy: 94.1870
Training Precision Scores: [41.67497507 99.87050826]
Training Recall Scores: [97.20930233 94.05487805]
Training F1 Scores: [58.33914864 96.87549066]

Validation Accuracy: 91.9003
Validation Precision Scores: [27.62645914 99.04803116]
Validation Recall Scores: [76.34408602 92.48484848]
Validation F1 Scores: [40.57142857 95.65399081]

Test Accuracy: 92.8290
Test Precision Scores: [29.04689864 99.13277512]
Test Recall Scores: [76.8       93.3934357]
Test F1 Scores: [42.15148189 96.17755857]

Out[13]:
<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x79a6a00ed850>
No description has been provided for this image

Task 3: RNN¶

In [9]:
import random
import time
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, ConfusionMatrixDisplay, precision_score, f1_score, recall_score
import torch
from sklearn.model_selection import train_test_split
from torch import nn, optim
from torch.utils.data import TensorDataset, DataLoader, Dataset
from sklearn.utils import compute_class_weight
from tqdm.notebook import tqdm
import matplotlib.pyplot as plt
from torch.nn.utils.rnn import pad_sequence,  pack_padded_sequence

# Set seeds
seed = 69
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)

# Set GPU
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"The device is set to: {device}")
The device is set to: cuda

Preprocessing¶

In [10]:
data_1 = pd.read_json("/kaggle/input/lab22222/train.json").drop(columns='md5hash')
data_2 = pd.read_json("/kaggle/input/lab22222/test.json").drop(columns='md5hash')

previous_length = len(data_1)
data_1 = data_1.drop_duplicates(subset=["api_call_sequence"])
print(f"Removed {previous_length - len(data_1)} duplicates \nDataset length: {len(data_1)}")

previous_length = len(data_2)
data_2 = data_2.drop_duplicates(subset=["api_call_sequence"])
print(f"Removed {previous_length - len(data_2)} duplicates \nDataset length: {len(data_2)}")

X = data_1.iloc[:, :-1].values # Remove label column
X = [x[0] for x in X]
y = data_1.iloc[:, -1].values

X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=seed)  # 80% of the data is used for training
X_test = data_2.iloc[:, :-1].values # Remove label column
X_test = [x[0] for x in X_test]
y_test = data_2.iloc[:, -1].values

X_train_str = [str(x) for x in X_train]
X_val_str = [str(x) for x in X_val]
X_test_str = [str(x) for x in X_test]

vectorizer = CountVectorizer()

# Computing the matrix representing the words contained in each api sequence
document_term_matrix_training = vectorizer.fit_transform(X_train_str)  # The vocabulary is computed on the training set and is used to compute the matrix for the other datasets
Removed 17875 duplicates 
Dataset length: 12838
Removed 5814 duplicates 
Dataset length: 7349
In [11]:
# Assigning sequential IDs to training set
for i in range(0, len(X_train)):
    row = X_train[i]
    for j in range(len(row)):
        row[j] = vectorizer.vocabulary_[row[j].lower()]
    X_train[i] = row

# Assigning sequential IDs to validation set
for i in range(0, len(X_val)):
    row = X_val[i]
    for j in range(len(row)):
        if row[j].lower() not in vectorizer.vocabulary_.keys():  # To manage the out-of-vocabulary words
            row[j] = max(vectorizer.vocabulary_.values()) + 2
        else:
            row[j] = vectorizer.vocabulary_[row[j].lower()]
    X_val[i] = row

# Assigning sequential IDs to testing set
for i in range(0, len(X_test)):
    row = X_test[i]
    for j in range(len(row)):
        if row[j].lower() not in vectorizer.vocabulary_.keys():  # To manage the out-of-vocabulary words
            row[j] = max(vectorizer.vocabulary_.values()) + 2
        else:
            row[j] = vectorizer.vocabulary_[row[j].lower()]
    X_test[i] = row
In [12]:
class TimeSeriesDataset(Dataset):
    def __init__(self, time_series_list, labels_list):
        """
        Initialize the dataset with sequences and corresponding labels.

        Args:
            time_series_list: List of time series sequences
            labels_list: List of corresponding labels
        """
        super(TimeSeriesDataset, self).__init__()
        self.time_series_list = time_series_list  # List of sequences
        self.labels_list = labels_list            # List of labels

    def __len__(self):
        """Returns the total number of samples in the dataset"""
        return len(self.time_series_list)

    def __getitem__(self, idx):
        """
        Retrieves a single sample and its label.

        Args:
            idx: Index of the sample to retrieve

        Returns:
            tuple: (sequence tensor, label tensor)
        """
        sequence = torch.tensor(self.time_series_list[idx], dtype=torch.float32)
        label = torch.tensor(self.labels_list[idx], dtype=torch.long)
        return sequence, label

def collate_fn(batch):
    """
    Custom collate function for DataLoader to handle variable-length sequences.
    Sorts sequences by length (descending) for efficient RNN processing and pads them.

    Args:
        batch: List of (sequence, label) tuples

    Returns:
        tuple: (padded_sequences, sequence_lengths, sorted_labels)
    """
    sequences, labels = zip(*batch) # Unpack batch
    lengths = torch.tensor([len(seq) for seq in sequences]) # Sequence lengths

    # Sort sequences by length (descending) to optimize RNN processing
    sorted_indices = lengths.argsort(descending=True)
    sequences = [sequences[i] for i in sorted_indices]
    labels = torch.stack([labels[i] for i in sorted_indices])  # Convert to tensor
    lengths = lengths[sorted_indices]  # Sort lengths too

    # Pad sequences to have uniform length in a batch
    padded_sequences = pad_sequence(sequences, batch_first=True, padding_value=max(vectorizer.vocabulary_.values()) + 1)
    return padded_sequences, lengths, labels
In [13]:
def training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs, patience=10, min_delta=0.001):
    start_time = time.time()
    train_losses = []
    val_losses = []

    best_val_loss = float('inf')
    epochs_no_improvement = 0

    # Training loop
    for epoch in tqdm(range(num_epochs), desc="Training"):  # Train for epochs
        train_loss = 0
        val_loss = 0
        model.train()  # Set model to training mode
        for batch_X, lengths, batch_y in train_loader:
            batch_X, lengths, batch_y = batch_X.to(device), lengths.to(device), batch_y.to(device)  # Move batch to GPU
            optimizer.zero_grad()  # Clear previous gradients
            outputs = model(batch_X,lengths)  # Forward pass
            loss = criterion(outputs, batch_y)  # Compute loss
            loss.backward()  # Backpropagation
            optimizer.step()  # Update model parameters
            train_loss += loss.item() * batch_X.size(0)
        train_loss /= len(train_dataset)
        train_losses.append(train_loss)  # Store training loss

        # Compute Validation loss during training
        model.eval()  # Set model to evaluation mode
        with torch.no_grad():  # Disable gradient computation for validation
            for batch_X, lengths, batch_y in val_loader:
                batch_X, lengths, batch_y = batch_X.to(device), lengths.to(device), batch_y.to(device)  # Move batch to GPU
                val_outputs = model(batch_X,lengths)  # Forward pass
                loss = criterion(val_outputs, batch_y)  # Compute validation loss
                val_loss += loss.item() * batch_X.size(0)
            val_loss /= len(val_dataset)
            val_losses.append(val_loss)  # Store validation loss

        # Early stopping
        if best_val_loss - val_loss > min_delta:
            best_val_loss = val_loss
            epochs_no_improvement = 0
            torch.save(model.state_dict(), 'best_model.pth')  # Saving the best model
        else:
            epochs_no_improvement += 1
            if epochs_no_improvement >= patience:
                print(f'Early stopping triggered after {epoch + 1} epochs because no improvements over {patience} subsequent epochs!')
                print(f'Epoch {epoch+1}/{num_epochs}, Train Loss: {train_losses[-1]:.4f}, Val Loss: {val_losses[-1]:.4f}')
                break  # Stop training

        if (epoch + 1) % 20 == 0:
            print(f'Epoch {epoch+1}/{num_epochs}, Train Loss: {train_losses[-1]:.4f}, Val Loss: {val_losses[-1]:.4f}')

    end_time = time.time()
    elapsed_time = end_time - start_time

    print(f'The function took {elapsed_time:.2f} seconds to execute.')

    # Restore best model weights
    model.load_state_dict(torch.load('best_model.pth', map_location=device))

    # Plot the training and validation loss
    plt.figure(figsize=(10, 5))
    plt.plot(train_losses, label='Train Loss')
    plt.plot(val_losses, label='Validation Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.title('Training and Validation Loss')
    plt.legend()
    plt.grid()
    plt.show()

    return

def testing_model(model, dataloader, device):
    model.eval()  # Set the model to evaluation mode
    all_labels = []
    all_predictions = []

    with torch.no_grad():  # Disable gradient computation for efficiency
        for inputs, lengths, labels in dataloader:
            inputs, lengths, labels = inputs.to(device), lengths.to(device), labels.to(device)  # Move batch to GPU
            outputs = model(inputs,lengths)
            _, predicted = torch.max(outputs, 1)  # Get the class with the highest score
            all_labels.extend(labels.cpu().numpy())
            all_predictions.extend(predicted.cpu().numpy())

    accuracy = accuracy_score(all_labels, all_predictions) * 100
    precision = precision_score(all_labels, all_predictions, average=None) * 100
    recall = recall_score(all_labels, all_predictions, average=None) * 100
    f1 = f1_score(all_labels, all_predictions, average=None) * 100

    return accuracy, precision, recall, f1, all_labels, all_predictions

def stats(accuracy, precision, recall, f1, description):
    print(f'{description} Accuracy: {accuracy:.4f}')
    print(f'{description} Precision Scores: {precision}')
    print(f'{description} Recall Scores: {recall}')
    print(f'{description} F1 Scores: {f1}\n')

Monodirectional¶

In [19]:
class MonoDirectionalRNN(nn.Module):
    def __init__(self, vocab_size, embed_size, hid_size, out_size):
        super(MonoDirectionalRNN, self).__init__()

        # 1) embedding layer: vocab_size × d
        self.embedding = nn.Embedding(vocab_size, embed_size, padding_idx=max(vectorizer.vocabulary_.values()) + 1)

        self.hidden_layer_size = hid_size

        # 2) First linear layer: (seq_len·d) → H
        # batch first is used to specify that the matrix in input has dimension: (batch_size, sequence_length, input_size)
        self.rnn = nn.RNN(input_size=embed_size, hidden_size=hid_size, batch_first=True, num_layers=2, nonlinearity="tanh", dropout=0.5)

        # 3) secondo layer lineare: H → 2
        self.linear = nn.Linear(hid_size, out_size)

    """
     Output of rnn:
         - **rnn_out** contains the output for all time steps.
           It has shape (batch_size, sequence_length, hidden_size).
         - **h** contains the hidden states for all layers at the last time step.A
            It has shape (num_layers, batch_size, hidden_size).
         - if different time-series have different sequence_length, rnn_out can generate errors
         - with h we get batch_size instead of sequence_length
           ==> no problem with different sequence_length

         - We do not return them in the tracking loop to
             - avoid problems with the gradient vanish/exploding
             - use the windows independently (if possible) and shuffle the data
        Returns:
         - predictions: The predicted pollution levels for the input sequence.
         Use h from last hidden node helps with time-series having different lengths
    """
    def forward(self, x: torch.LongTensor, lengths):

        # embedding lookup → (B, seq_len, d) # certo, perchè se prima avevo per ogni parola d dim, se qui ho 100 parole avrò 100xd dimensioni
        x = self.embedding(x.long())

        x = pack_padded_sequence(x, lengths.cpu(), batch_first=True, enforce_sorted=True)   # internally it knows, thanks to collate_fn function, which ones are padded

        rnn_out, h = self.rnn(x)

        predictions = self.linear(h[-1])
        return predictions
In [20]:
train_dataset = TimeSeriesDataset(X_train, y_train)
val_dataset = TimeSeriesDataset(X_val, y_val)
test_dataset = TimeSeriesDataset(X_test, y_test)

train_loader = DataLoader(train_dataset, batch_size=64, collate_fn=collate_fn,shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)

embedding_sizes = [4,8,16,32,64,128,256]
hidden_dims = [8,16,32,64,128,256,320,384,512]

for i in range(len(embedding_sizes)):
    for j in range(i,len(hidden_dims)):
        embedding_size=embedding_sizes[i]
        hidden_dim=hidden_dims[j]
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print(f"Trying with embedding size {embedding_size} and hidden dim {hidden_dim}")

        vocabulary_size = len(vectorizer.vocabulary_) + 2  # Add 2 for out-of-vocabulary and padding
        output_size = 2
        
        model = MonoDirectionalRNN(vocabulary_size, embedding_size, hidden_dim, output_size)
        model = model.to(device)
        
        class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)  # Computing class weights
        class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
        criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
        
        optimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.01)
        num_epochs = 100
        training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs)
        
        train_accuracy, train_precisionscores, train_recallscores, train_f1scores, _, _ = testing_model(model,train_loader,device)
        val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model(model,val_loader,device)
        
        stats(train_accuracy, train_precisionscores, train_recallscores, train_f1scores, 'Training')
        stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 8
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5264, Val Loss: 0.6036
Epoch 40/100, Train Loss: 0.4578, Val Loss: 0.5140
Epoch 60/100, Train Loss: 0.4027, Val Loss: 0.4801
Early stopping triggered after 68 epochs because no improvements over 10 subsequent epochs!
Epoch 68/100, Train Loss: 0.3889, Val Loss: 0.4896
The function took 122.79 seconds to execute.
No description has been provided for this image
Training Accuracy: 78.8802
Training Precision Scores: [15.31711209 99.40744558]
Training Recall Scores: [89.30232558 78.42479675]
Training F1 Scores: [26.14913177 87.67823666]

Validation Accuracy: 78.5826
Validation Precision Scores: [12.72430669 99.23273657]
Validation Recall Scores: [83.87096774 78.38383838]
Validation F1 Scores: [22.09631728 87.58465011]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 16
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5278, Val Loss: 0.5982
Epoch 40/100, Train Loss: 0.4726, Val Loss: 0.5960
Epoch 60/100, Train Loss: 0.4490, Val Loss: 0.5663
Early stopping triggered after 69 epochs because no improvements over 10 subsequent epochs!
Epoch 69/100, Train Loss: 0.4747, Val Loss: 0.5710
The function took 124.98 seconds to execute.
No description has been provided for this image
Training Accuracy: 74.6446
Training Precision Scores: [13.10251188 99.39923539]
Training Recall Scores: [89.76744186 73.98373984]
Training F1 Scores: [22.86729858 84.82871126]

Validation Accuracy: 72.8193
Validation Precision Scores: [ 9.93377483 99.00717044]
Validation Recall Scores: [80.64516129 72.52525253]
Validation F1 Scores: [17.68867925 83.72201493]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 32
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5369, Val Loss: 0.5741
Early stopping triggered after 37 epochs because no improvements over 10 subsequent epochs!
Epoch 37/100, Train Loss: 0.5240, Val Loss: 0.5501
The function took 66.35 seconds to execute.
No description has been provided for this image
Training Accuracy: 75.9883
Training Precision Scores: [12.79239766 98.93814707]
Training Recall Scores: [81.39534884 75.75203252]
Training F1 Scores: [22.10991788 85.80637735]

Validation Accuracy: 75.5841
Validation Precision Scores: [10.26785714 98.73417722]
Validation Recall Scores: [74.19354839 75.63636364]
Validation F1 Scores: [18.03921569 85.65545642]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5217, Val Loss: 0.5784
Early stopping triggered after 23 epochs because no improvements over 10 subsequent epochs!
Epoch 23/100, Train Loss: 0.5247, Val Loss: 0.5611
The function took 41.30 seconds to execute.
No description has been provided for this image
Training Accuracy: 68.7926
Training Precision Scores: [10.14076415 98.86581234]
Training Recall Scores: [82.09302326 68.21138211]
Training F1 Scores: [18.05164919 80.72644176]

Validation Accuracy: 67.9128
Validation Precision Scores: [ 8.13287514 98.7020649 ]
Validation Recall Scores: [76.34408602 67.5959596 ]
Validation F1 Scores: [14.69979296 80.23980815]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 128
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.6701, Val Loss: 0.6778
Early stopping triggered after 26 epochs because no improvements over 10 subsequent epochs!
Epoch 26/100, Train Loss: 0.6693, Val Loss: 0.6487
The function took 52.18 seconds to execute.
No description has been provided for this image
Training Accuracy: 63.3398
Training Precision Scores: [ 6.78932366 97.37950398]
Training Recall Scores: [60.93023256 63.44512195]
Training F1 Scores: [12.21730007 76.83219494]

Validation Accuracy: 62.3832
Validation Precision Scores: [ 6.13065327 97.96567069]
Validation Recall Scores: [65.59139785 62.26262626]
Validation F1 Scores: [11.21323529 76.13636364]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 256
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 11 epochs because no improvements over 10 subsequent epochs!
Epoch 11/100, Train Loss: 0.7072, Val Loss: 0.7769
The function took 23.50 seconds to execute.
No description has been provided for this image
Training Accuracy: 61.2950
Training Precision Scores: [ 5.34139582 96.54023171]
Training Recall Scores: [49.30232558 61.81910569]
Training F1 Scores: [ 9.63855422 75.37327303]

Validation Accuracy: 62.3832
Validation Precision Scores: [ 5.23076923 97.36346516]
Validation Recall Scores: [54.83870968 62.66666667]
Validation F1 Scores: [ 9.5505618  76.25368732]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 320
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 20 epochs because no improvements over 10 subsequent epochs!
Epoch 20/100, Train Loss: 0.7095, Val Loss: 0.7094
The function took 46.25 seconds to execute.
No description has been provided for this image
Training Accuracy: 91.2171
Training Precision Scores: [10.53511706 96.20554177]
Training Recall Scores: [14.65116279 94.56300813]
Training F1 Scores: [12.25680934 95.37720377]

Validation Accuracy: 91.0826
Validation Precision Scores: [ 9.03614458 96.75270608]
Validation Recall Scores: [16.12903226 93.8989899 ]
Validation F1 Scores: [11.58301158 95.30449047]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 384
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 20 epochs because no improvements over 10 subsequent epochs!
Epoch 20/100, Train Loss: 0.7065, Val Loss: 0.7233
The function took 49.96 seconds to execute.
No description has been provided for this image
/usr/local/lib/python3.11/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
Training Accuracy: 95.8130
Training Precision Scores: [ 0.         95.81304771]
Training Recall Scores: [  0. 100.]
Training F1 Scores: [ 0.         97.86176032]

Validation Accuracy: 96.3785
Validation Precision Scores: [ 0.         96.37850467]
Validation Recall Scores: [  0. 100.]
Validation F1 Scores: [ 0.         98.15585961]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 512
/usr/local/lib/python3.11/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.7350, Val Loss: 0.7335
Early stopping triggered after 22 epochs because no improvements over 10 subsequent epochs!
Epoch 22/100, Train Loss: 0.7337, Val Loss: 0.6847
The function took 70.65 seconds to execute.
No description has been provided for this image
/usr/local/lib/python3.11/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
Training Accuracy: 95.8130
Training Precision Scores: [ 0.         95.81304771]
Training Recall Scores: [  0. 100.]
Training F1 Scores: [ 0.         97.86176032]

Validation Accuracy: 96.3785
Validation Precision Scores: [ 0.         96.37850467]
Validation Recall Scores: [  0. 100.]
Validation F1 Scores: [ 0.         98.15585961]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 16
/usr/local/lib/python3.11/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5037, Val Loss: 0.5322
Early stopping triggered after 22 epochs because no improvements over 10 subsequent epochs!
Epoch 22/100, Train Loss: 0.4966, Val Loss: 0.5785
The function took 39.86 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.4557
Training Precision Scores: [18.15772034 98.21938135]
Training Recall Scores: [63.72093023 87.44918699]
Training F1 Scores: [28.26199072 92.52190742]

Validation Accuracy: 86.4875
Validation Precision Scores: [14.72222222 98.1884058 ]
Validation Recall Scores: [56.98924731 87.5959596 ]
Validation F1 Scores: [23.3995585  92.59021994]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 32
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 16 epochs because no improvements over 10 subsequent epochs!
Epoch 16/100, Train Loss: 0.4756, Val Loss: 0.5743
The function took 28.92 seconds to execute.
No description has been provided for this image
Training Accuracy: 72.5998
Training Precision Scores: [11.223162   98.81878822]
Training Recall Scores: [80.23255814 72.26626016]
Training F1 Scores: [19.69178082 83.48203804]

Validation Accuracy: 71.1449
Validation Precision Scores: [ 8.46153846 98.48993289]
Validation Recall Scores: [70.96774194 71.15151515]
Validation F1 Scores: [15.12027491 82.61787474]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 19 epochs because no improvements over 10 subsequent epochs!
Epoch 19/100, Train Loss: 0.6490, Val Loss: 0.6160
The function took 34.27 seconds to execute.
No description has been provided for this image
Training Accuracy: 73.8169
Training Precision Scores: [11.46366428 98.71917155]
Training Recall Scores: [78.13953488 73.62804878]
Training F1 Scores: [19.99404939 84.34716805]

Validation Accuracy: 73.5202
Validation Precision Scores: [ 9.17941586 98.53975122]
Validation Recall Scores: [70.96774194 73.61616162]
Validation F1 Scores: [16.25615764 84.27382054]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 128
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 15 epochs because no improvements over 10 subsequent epochs!
Epoch 15/100, Train Loss: 0.6116, Val Loss: 0.6224
The function took 29.91 seconds to execute.
No description has been provided for this image
Training Accuracy: 67.0594
Training Precision Scores: [ 8.67618248 98.2081529 ]
Training Recall Scores: [72.09302326 66.83943089]
Training F1 Scores: [15.48838371 79.54284332]

Validation Accuracy: 65.9657
Validation Precision Scores: [ 6.65926748 98.02039592]
Validation Recall Scores: [64.51612903 66.02020202]
Validation F1 Scores: [12.07243461 78.89908257]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 256
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 11 epochs because no improvements over 10 subsequent epochs!
Epoch 11/100, Train Loss: 0.7061, Val Loss: 0.6923
The function took 23.63 seconds to execute.
No description has been provided for this image
Training Accuracy: 70.3992
Training Precision Scores: [ 6.17864338 96.62643993]
Training Recall Scores: [42.79069767 71.60569106]
Training F1 Scores: [10.79812207 82.25542844]

Validation Accuracy: 69.0031
Validation Precision Scores: [ 4.64516129 96.82097044]
Validation Recall Scores: [38.70967742 70.14141414]
Validation F1 Scores: [ 8.29493088 81.34957826]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 320
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 11 epochs because no improvements over 10 subsequent epochs!
Epoch 11/100, Train Loss: 0.6775, Val Loss: 0.6805
The function took 25.42 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.4265
Training Precision Scores: [ 9.22165821 96.46786972]
Training Recall Scores: [25.34883721 89.09552846]
Training F1 Scores: [13.5235732  92.63524937]

Validation Accuracy: 86.9938
Validation Precision Scores: [ 7.11743772 96.80804547]
Validation Recall Scores: [21.50537634 89.45454545]
Validation F1 Scores: [10.69518717 92.98614028]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 384
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 11 epochs because no improvements over 10 subsequent epochs!
Epoch 11/100, Train Loss: 0.7177, Val Loss: 0.7133
The function took 27.48 seconds to execute.
No description has been provided for this image
Training Accuracy: 83.5540
Training Precision Scores: [11.54551008 97.20838642]
Training Recall Scores: [43.95348837 85.28455285]
Training F1 Scores: [18.287373   90.85692632]

Validation Accuracy: 82.8271
Validation Precision Scores: [ 8.17307692 97.25836431]
Validation Recall Scores: [36.55913978 84.56565657]
Validation F1 Scores: [13.35952849 90.46898638]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 17 epochs because no improvements over 10 subsequent epochs!
Epoch 17/100, Train Loss: 0.7261, Val Loss: 0.7342
The function took 54.35 seconds to execute.
No description has been provided for this image
Training Accuracy: 84.7030
Training Precision Scores: [ 4.54183267 95.86245147]
Training Recall Scores: [13.25581395 87.82520325]
Training F1 Scores: [ 6.76557864 91.66799257]

Validation Accuracy: 85.1246
Validation Precision Scores: [ 4.98442368 96.57320872]
Validation Recall Scores: [17.20430108 87.67676768]
Validation F1 Scores: [ 7.7294686  91.91020754]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 32
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5090, Val Loss: 0.5262
Epoch 40/100, Train Loss: 0.5002, Val Loss: 0.5616
Early stopping triggered after 59 epochs because no improvements over 10 subsequent epochs!
Epoch 59/100, Train Loss: 0.4319, Val Loss: 0.5056
The function took 106.79 seconds to execute.
No description has been provided for this image
Training Accuracy: 81.0224
Training Precision Scores: [16.58600968 99.33725147]
Training Recall Scores: [87.6744186  80.73170732]
Training F1 Scores: [27.89493156 89.07327465]

Validation Accuracy: 80.3738
Validation Precision Scores: [12.83905967 98.90818859]
Validation Recall Scores: [76.34408602 80.52525253]
Validation F1 Scores: [21.98142415 88.77505568]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3999, Val Loss: 0.5504
Early stopping triggered after 27 epochs because no improvements over 10 subsequent epochs!
Epoch 27/100, Train Loss: 0.3650, Val Loss: 0.5606
The function took 48.50 seconds to execute.
No description has been provided for this image
Training Accuracy: 78.9484
Training Precision Scores: [15.47049442 99.45890234]
Training Recall Scores: [90.23255814 78.45528455]
Training F1 Scores: [26.41252553 87.71730485]

Validation Accuracy: 78.3489
Validation Precision Scores: [11.35225376 98.73031996]
Validation Recall Scores: [73.11827957 78.54545455]
Validation F1 Scores: [19.65317919 87.48874887]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 128
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 17 epochs because no improvements over 10 subsequent epochs!
Epoch 17/100, Train Loss: 0.6015, Val Loss: 0.6350
The function took 34.73 seconds to execute.
No description has been provided for this image
Training Accuracy: 77.5852
Training Precision Scores: [12.61980831 98.53206284]
Training Recall Scores: [73.48837209 77.76422764]
Training F1 Scores: [21.54055896 86.92491196]

Validation Accuracy: 76.8692
Validation Precision Scores: [10.42654028 98.60465116]
Validation Recall Scores: [70.96774194 77.09090909]
Validation F1 Scores: [18.18181818 86.53061224]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 256
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5524, Val Loss: 0.5688
Early stopping triggered after 31 epochs because no improvements over 10 subsequent epochs!
Epoch 31/100, Train Loss: 0.5593, Val Loss: 0.5685
The function took 66.36 seconds to execute.
No description has been provided for this image
Training Accuracy: 73.8559
Training Precision Scores: [11.94735066 98.95990146]
Training Recall Scores: [82.3255814  73.48577236]
Training F1 Scores: [20.86648983 84.34128419]

Validation Accuracy: 72.5857
Validation Precision Scores: [ 9.64332893 98.89563777]
Validation Recall Scores: [78.49462366 72.36363636]
Validation F1 Scores: [17.17647059 83.57442837]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 320
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 20 epochs because no improvements over 10 subsequent epochs!
Epoch 20/100, Train Loss: 0.6478, Val Loss: 0.6867
The function took 46.19 seconds to execute.
No description has been provided for this image
Training Accuracy: 75.3554
Training Precision Scores: [10.13282732 97.86509496]
Training Recall Scores: [62.09302326 75.93495935]
Training F1 Scores: [17.42251223 85.51645207]

Validation Accuracy: 75.3115
Validation Precision Scores: [ 8.44854071 98.01773605]
Validation Recall Scores: [59.13978495 75.91919192]
Validation F1 Scores: [14.78494624 85.56466302]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 384
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 11 epochs because no improvements over 10 subsequent epochs!
Epoch 11/100, Train Loss: 0.7091, Val Loss: 0.6937
The function took 27.40 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.9523
Training Precision Scores: [10.0877193  96.54983571]
Training Recall Scores: [26.74418605 89.58333333]
Training F1 Scores: [14.64968153 92.93621508]

Validation Accuracy: 86.6044
Validation Precision Scores: [ 7.74410774 96.91765742]
Validation Recall Scores: [24.7311828  88.92929293]
Validation F1 Scores: [11.79487179 92.75179098]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 17 epochs because no improvements over 10 subsequent epochs!
Epoch 17/100, Train Loss: 0.7044, Val Loss: 0.6906
The function took 54.39 seconds to execute.
No description has been provided for this image
Training Accuracy: 89.1334
Training Precision Scores: [ 7.96568627 96.13920034]
Training Recall Scores: [15.11627907 92.36788618]
Training F1 Scores: [10.43338684 94.21581839]

Validation Accuracy: 88.8240
Validation Precision Scores: [ 5.90909091 96.59284497]
Validation Recall Scores: [13.97849462 91.63636364]
Validation F1 Scores: [ 8.30670927 94.04934688]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3621, Val Loss: 0.5710
Early stopping triggered after 36 epochs because no improvements over 10 subsequent epochs!
Epoch 36/100, Train Loss: 0.2202, Val Loss: 0.5258
The function took 64.82 seconds to execute.
No description has been provided for this image
Training Accuracy: 85.3846
Training Precision Scores: [21.59151194 99.72570066]
Training Recall Scores: [94.65116279 84.9796748 ]
Training F1 Scores: [35.16198704 91.76406036]

Validation Accuracy: 84.2679
Validation Precision Scores: [15.82417582 99.00615239]
Validation Recall Scores: [77.41935484 84.52525253]
Validation F1 Scores: [26.27737226 91.19442023]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 128
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5210, Val Loss: 0.8151
Early stopping triggered after 36 epochs because no improvements over 10 subsequent epochs!
Epoch 36/100, Train Loss: 0.5345, Val Loss: 0.6026
The function took 71.34 seconds to execute.
No description has been provided for this image
Training Accuracy: 79.4158
Training Precision Scores: [14.20068027 98.78757262]
Training Recall Scores: [77.6744186  79.49186992]
Training F1 Scores: [24.01150252 88.09550625]

Validation Accuracy: 78.8551
Validation Precision Scores: [11.86440678 98.8372093 ]
Validation Recall Scores: [75.2688172  78.98989899]
Validation F1 Scores: [20.49780381 87.8059735 ]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 256
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.6673, Val Loss: 0.6007
Early stopping triggered after 32 epochs because no improvements over 10 subsequent epochs!
Epoch 32/100, Train Loss: 0.5245, Val Loss: 0.5575
The function took 68.49 seconds to execute.
No description has been provided for this image
Training Accuracy: 82.4830
Training Precision Scores: [14.40457618 98.16700611]
Training Recall Scores: [64.41860465 83.27235772]
Training F1 Scores: [23.54441139 90.10831913]

Validation Accuracy: 82.7882
Validation Precision Scores: [12.79317697 98.42782277]
Validation Recall Scores: [64.51612903 83.47474747]
Validation F1 Scores: [21.35231317 90.33668561]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 320
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.6055, Val Loss: 0.6328
Early stopping triggered after 36 epochs because no improvements over 10 subsequent epochs!
Epoch 36/100, Train Loss: 0.6901, Val Loss: 0.5833
The function took 83.43 seconds to execute.
No description has been provided for this image
Training Accuracy: 69.7566
Training Precision Scores: [10.71638285 99.0530303 ]
Training Recall Scores: [84.88372093 69.09552846]
Training F1 Scores: [19.03023983 81.40565134]

Validation Accuracy: 68.9642
Validation Precision Scores: [ 8.58823529 98.83585565]
Validation Recall Scores: [78.49462366 68.60606061]
Validation F1 Scores: [15.48250265 80.99212974]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 384
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.7160, Val Loss: 0.6993
Early stopping triggered after 25 epochs because no improvements over 10 subsequent epochs!
Epoch 25/100, Train Loss: 0.7216, Val Loss: 0.7196
The function took 62.64 seconds to execute.
No description has been provided for this image
Training Accuracy: 72.3174
Training Precision Scores: [ 4.62579917 95.96636447]
Training Recall Scores: [28.60465116 74.22764228]
Training F1 Scores: [ 7.96374231 83.70866999]

Validation Accuracy: 73.9097
Validation Precision Scores: [ 4.8513302  96.78589943]
Validation Recall Scores: [33.33333333 75.43434343]
Validation F1 Scores: [ 8.46994536 84.78655767]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 11 epochs because no improvements over 10 subsequent epochs!
Epoch 11/100, Train Loss: 0.7354, Val Loss: 0.8606
The function took 35.27 seconds to execute.
No description has been provided for this image
Training Accuracy: 90.6524
Training Precision Scores: [ 8.59375  96.105919]
Training Recall Scores: [12.79069767 94.05487805]
Training F1 Scores: [10.28037383 95.06933744]

Validation Accuracy: 90.6931
Validation Precision Scores: [ 8.98876404 96.77824268]
Validation Recall Scores: [17.20430108 93.45454545]
Validation F1 Scores: [11.80811808 95.08735868]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 128
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4163, Val Loss: 0.6459
Early stopping triggered after 37 epochs because no improvements over 10 subsequent epochs!
Epoch 37/100, Train Loss: 0.2135, Val Loss: 0.6829
The function took 73.53 seconds to execute.
No description has been provided for this image
Training Accuracy: 84.4596
Training Precision Scores: [20.67404427 99.77058681]
Training Recall Scores: [95.58139535 83.97357724]
Training F1 Scores: [33.99503722 91.19302505]

Validation Accuracy: 84.1121
Validation Precision Scores: [15.38461538 98.91150024]
Validation Recall Scores: [75.2688172  84.44444444]
Validation F1 Scores: [25.54744526 91.10723627]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 256
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 11 epochs because no improvements over 10 subsequent epochs!
Epoch 11/100, Train Loss: 0.7137, Val Loss: 0.6826
The function took 23.84 seconds to execute.
No description has been provided for this image
Training Accuracy: 72.0156
Training Precision Scores: [ 8.82749326 97.69926048]
Training Recall Scores: [60.93023256 72.5       ]
Training F1 Scores: [15.42083579 83.23416171]

Validation Accuracy: 72.0794
Validation Precision Scores: [ 7.02479339 97.71986971]
Validation Recall Scores: [54.83870968 72.72727273]
Validation F1 Scores: [12.45421245 83.39124392]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 320
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 11 epochs because no improvements over 10 subsequent epochs!
Epoch 11/100, Train Loss: 0.7111, Val Loss: 0.6905
The function took 25.70 seconds to execute.
No description has been provided for this image
Training Accuracy: 81.2074
Training Precision Scores: [11.85147508 97.62764933]
Training Recall Scores: [54.18604651 82.38821138]
Training F1 Scores: [19.4490818  89.36287478]

Validation Accuracy: 80.4517
Validation Precision Scores: [ 9.34393638 97.77239709]
Validation Recall Scores: [50.53763441 81.57575758]
Validation F1 Scores: [15.77181208 88.94273128]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 384
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 16 epochs because no improvements over 10 subsequent epochs!
Epoch 16/100, Train Loss: 0.6881, Val Loss: 0.6850
The function took 40.26 seconds to execute.
No description has been provided for this image
Training Accuracy: 73.7001
Training Precision Scores: [ 9.46090682 97.79086892]
Training Recall Scores: [61.62790698 74.22764228]
Training F1 Scores: [16.40359022 84.39540124]

Validation Accuracy: 72.0405
Validation Precision Scores: [ 6.04781997 97.30748519]
Validation Recall Scores: [46.23655914 73.01010101]
Validation F1 Scores: [10.69651741 83.42566944]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.7126, Val Loss: 0.7020
Early stopping triggered after 23 epochs because no improvements over 10 subsequent epochs!
Epoch 23/100, Train Loss: 0.7401, Val Loss: 0.7429
The function took 73.86 seconds to execute.
No description has been provided for this image
Training Accuracy: 50.9640
Training Precision Scores: [ 4.05027933 95.68276911]
Training Recall Scores: [47.20930233 51.12804878]
Training F1 Scores: [ 7.46049247 66.64458869]

Validation Accuracy: 53.9330
Validation Precision Scores: [ 4.50751252 97.15328467]
Validation Recall Scores: [58.06451613 53.77777778]
Validation F1 Scores: [ 8.36560806 69.23276983]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 256
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 13 epochs because no improvements over 10 subsequent epochs!
Epoch 13/100, Train Loss: 0.6749, Val Loss: 0.7803
The function took 28.68 seconds to execute.
No description has been provided for this image
Training Accuracy: 63.9533
Training Precision Scores: [ 8.58227848 98.56012658]
Training Recall Scores: [78.8372093  63.30284553]
Training F1 Scores: [15.47945205 77.09158416]

Validation Accuracy: 63.3178
Validation Precision Scores: [ 6.9035533  98.42072015]
Validation Recall Scores: [73.11827957 62.94949495]
Validation F1 Scores: [12.61595547 76.78659438]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 320
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5542, Val Loss: 0.5667
Early stopping triggered after 38 epochs because no improvements over 10 subsequent epochs!
Epoch 38/100, Train Loss: 0.6309, Val Loss: 0.5530
The function took 90.81 seconds to execute.
No description has been provided for this image
Training Accuracy: 75.0438
Training Precision Scores: [12.48680971 98.990171  ]
Training Recall Scores: [82.55813953 74.71544715]
Training F1 Scores: [21.69263672 85.15665721]

Validation Accuracy: 74.7664
Validation Precision Scores: [10.41369472 98.92876272]
Validation Recall Scores: [78.49462366 74.62626263]
Validation F1 Scores: [18.38790932 85.07600184]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 384
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 17 epochs because no improvements over 10 subsequent epochs!
Epoch 17/100, Train Loss: 0.7153, Val Loss: 0.7114
The function took 43.92 seconds to execute.
No description has been provided for this image
Training Accuracy: 76.6115
Training Precision Scores: [11.78294574 98.36150845]
Training Recall Scores: [70.69767442 76.8699187 ]
Training F1 Scores: [20.19933555 86.29777524]

Validation Accuracy: 76.5576
Validation Precision Scores: [10.17214397 98.54847071]
Validation Recall Scores: [69.89247312 76.80808081]
Validation F1 Scores: [17.75956284 86.33060854]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.6072, Val Loss: 0.7096
Early stopping triggered after 33 epochs because no improvements over 10 subsequent epochs!
Epoch 33/100, Train Loss: 0.5586, Val Loss: 0.5679
The function took 108.42 seconds to execute.
No description has been provided for this image
Training Accuracy: 79.3476
Training Precision Scores: [13.03017053 98.34648628]
Training Recall Scores: [69.30232558 79.78658537]
Training F1 Scores: [21.93595878 88.09964652]

Validation Accuracy: 78.3879
Validation Precision Scores: [10.17241379 98.28973843]
Validation Recall Scores: [63.44086022 78.94949495]
Validation F1 Scores: [17.53343239 87.56441855]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 320
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 17 epochs because no improvements over 10 subsequent epochs!
Epoch 17/100, Train Loss: 0.4967, Val Loss: 0.6197
The function took 42.07 seconds to execute.
No description has been provided for this image
Training Accuracy: 77.8092
Training Precision Scores: [13.61668634 98.91318411]
Training Recall Scores: [80.46511628 77.69308943]
Training F1 Scores: [23.29182094 87.02828846]

Validation Accuracy: 76.5966
Validation Precision Scores: [ 9.93690852 98.44881075]
Validation Recall Scores: [67.74193548 76.92929293]
Validation F1 Scores: [17.33149931 86.36879111]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 384
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5122, Val Loss: 0.6077
Early stopping triggered after 22 epochs because no improvements over 10 subsequent epochs!
Epoch 22/100, Train Loss: 0.4596, Val Loss: 0.6380
The function took 59.01 seconds to execute.
No description has been provided for this image
Training Accuracy: 77.7020
Training Precision Scores: [12.71050521 98.5468107 ]
Training Recall Scores: [73.72093023 77.87601626]
Training F1 Scores: [21.68262654 87.00045413]

Validation Accuracy: 76.8692
Validation Precision Scores: [10.04784689 98.45440495]
Validation Recall Scores: [67.74193548 77.21212121]
Validation F1 Scores: [17.5        86.54891304]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5964, Val Loss: 0.7276
Early stopping triggered after 34 epochs because no improvements over 10 subsequent epochs!
Epoch 34/100, Train Loss: 0.5583, Val Loss: 0.5655
The function took 114.57 seconds to execute.
No description has been provided for this image
Training Accuracy: 75.5794
Training Precision Scores: [11.48999259 98.41521395]
Training Recall Scores: [72.09302326 75.73170732]
Training F1 Scores: [19.82097187 85.59614059]

Validation Accuracy: 73.3645
Validation Precision Scores: [ 8.55539972 98.27493261]
Validation Recall Scores: [65.59139785 73.65656566]
Validation F1 Scores: [15.13647643 84.20323326]

In [21]:
# mettere e istanziare model con iperparam migliori Trying with embedding size 32 and hidden dim 64

train_dataset = TimeSeriesDataset(X_train, y_train)
val_dataset = TimeSeriesDataset(X_val, y_val)
test_dataset = TimeSeriesDataset(X_test, y_test)

train_loader = DataLoader(train_dataset, batch_size=64, collate_fn=collate_fn,shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)

embedding_size=32
hidden_dim=64

vocabulary_size = len(vectorizer.vocabulary_) + 2  # Add 2 for out-of-vocabulary and padding
output_size = 2

model = MonoDirectionalRNN(vocabulary_size, embedding_size, hidden_dim, output_size)
model = model.to(device)

class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)  # Computing class weights
class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)

optimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.01)
num_epochs = 100
training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs)

test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model(model,test_loader,device)
stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3642, Val Loss: 0.5233
Early stopping triggered after 25 epochs because no improvements over 10 subsequent epochs!
Epoch 25/100, Train Loss: 0.4786, Val Loss: 0.6104
The function took 45.15 seconds to execute.
No description has been provided for this image
Test Accuracy: 81.6029
Test Precision Scores: [12.61872456 98.9106383 ]
Test Recall Scores: [74.4        81.85659952]
Test F1 Scores: [21.57772622 89.57915832]

Out[21]:
<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x79a6b8b64c10>
No description has been provided for this image
No description has been provided for this image

Bidirectional¶

In [14]:
class BiDirectionalRNN(nn.Module):
    def __init__(self, vocab_size, embed_size, hid_size, out_size):
        super(BiDirectionalRNN, self).__init__()

        # 1) embedding layer: vocab_size × d
        self.embedding = nn.Embedding(vocab_size, embed_size, padding_idx=max(vectorizer.vocabulary_.values()) + 1)

        self.hidden_layer_size = hid_size

        # 2) first liner layer: (seq_len·d) → H
        #batch first is used to specify that the matrix in input has dimension: (batch_size, sequence_length, input_size)
        self.rnn = nn.RNN(input_size=embed_size, hidden_size=hid_size, batch_first=True, num_layers=2, dropout=0.5, bidirectional=True)

        # 3) second linear layer: H → 2
        self.linear = nn.Linear(hid_size * 2, out_size)

    """
    Args:
        - input_seq: The input sequence tensor of shape
         (batch_size, sequence_length, input_size).
     Output of rnn:
         - **rnn_out** contains the output for all time steps.
           It has shape (batch_size, sequence_length, hidden_size).
         - **h** contains the hidden states for all layers at the last time step.
            It has shape (num_layers, batch_size, hidden_size).
         - if different time-series have different sequence_length, rnn_out can generate errors
         - with h we get batch_size instead of sequence_length
           ==> no problem with different sequence_length

         - We do not return them in the tracking loop to
             - avoid problems with the gradient vanish/exploding
             - use the windows independently (if possible) and shuffle the data
        Returns:
         - predictions: The predicted pollution levels for the input sequence.
         Use h from last hidden node helps with time-series having different lengths
    """
    def forward(self, x: torch.LongTensor, lengths):
        # x è B,seq_len

        # embedding lookup → (B, seq_len, d) # certo, perchè se prima avevo per ogni parola d dim, se qui ho 100 parole avrò 100xd dimensioni
        x = self.embedding(x.long())
        x = pack_padded_sequence(x, lengths.cpu(), batch_first=True, enforce_sorted=True)

        rnn_out, h = self.rnn(x)

        # Concatenate the final forward and backward hidden states
        # hidden[-2] contains the last forward hidden state, hidden[-1] contains the last backward hidden state
        hidden_cat = torch.cat((h[-2], h[-1]), dim=1)

        predictions = self.linear(hidden_cat)
        return predictions
In [23]:
train_dataset = TimeSeriesDataset(X_train, y_train)
val_dataset = TimeSeriesDataset(X_val, y_val)
test_dataset = TimeSeriesDataset(X_test, y_test)

train_loader = DataLoader(train_dataset, batch_size=64, collate_fn=collate_fn,shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)

embedding_sizes = [4,8,16,32,64,128,256]
hidden_dims = [8,16,32,64,128,256,320,384,512]

for i in range(len(embedding_sizes)):
    for j in range(i,len(hidden_dims)):
        embedding_size=embedding_sizes[i]
        hidden_size=hidden_dims[j]
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print(f"Trying with embedding size {embedding_size} and hidden dim {hidden_dim}")

        vocabulary_size = len(vectorizer.vocabulary_) + 2  # Add 2 for out-of-vocabulary and padding
        output_size = 2

        model = BiDirectionalRNN(vocabulary_size, embedding_size, hidden_size, output_size)
        model = model.to(device)
        
        class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)  # Computing class weights
        class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
        criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
        
        optimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.01)
        num_epochs = 100
        training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs)
        
        train_accuracy, train_precisionscores, train_recallscores, train_f1scores, _, _ = testing_model(model,train_loader,device)
        val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model(model,val_loader,device)
        
        stats(train_accuracy, train_precisionscores, train_recallscores, train_f1scores, 'Training')
        stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4269, Val Loss: 0.4019
Epoch 40/100, Train Loss: 0.3609, Val Loss: 0.3569
Epoch 60/100, Train Loss: 0.3292, Val Loss: 0.3293
Early stopping triggered after 63 epochs because no improvements over 10 subsequent epochs!
Epoch 63/100, Train Loss: 0.3368, Val Loss: 0.3344
The function took 162.66 seconds to execute.
No description has been provided for this image
Training Accuracy: 87.9163
Training Precision Scores: [23.40983607 99.16523728]
Training Recall Scores: [83.02325581 88.1300813 ]
Training F1 Scores: [36.52173913 93.32257197]

Validation Accuracy: 87.7336
Validation Precision Scores: [20.4787234  99.27007299]
Validation Recall Scores: [82.79569892 87.91919192]
Validation F1 Scores: [32.8358209  93.25048211]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3817, Val Loss: 0.3548
Epoch 40/100, Train Loss: 0.3131, Val Loss: 0.3166
Early stopping triggered after 46 epochs because no improvements over 10 subsequent epochs!
Epoch 46/100, Train Loss: 0.3250, Val Loss: 0.3235
The function took 119.46 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.0078
Training Precision Scores: [21.37578169 99.36552697]
Training Recall Scores: [87.44186047 85.94512195]
Training F1 Scores: [34.35358611 92.16936407]

Validation Accuracy: 86.0592
Validation Precision Scores: [18.67612293 99.34731935]
Validation Recall Scores: [84.94623656 86.1010101 ]
Validation F1 Scores: [30.62015504 92.25108225]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3734, Val Loss: 0.3976
Early stopping triggered after 38 epochs because no improvements over 10 subsequent epochs!
Epoch 38/100, Train Loss: 0.2724, Val Loss: 0.3335
The function took 98.71 seconds to execute.
No description has been provided for this image
Training Accuracy: 87.3028
Training Precision Scores: [23.83233533 99.62790698]
Training Recall Scores: [92.55813953 87.07317073]
Training F1 Scores: [37.9047619  92.92841649]

Validation Accuracy: 86.9159
Validation Precision Scores: [18.9258312  99.12723932]
Validation Recall Scores: [79.56989247 87.19191919]
Validation F1 Scores: [30.5785124  92.77730009]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3720, Val Loss: 0.3803
Early stopping triggered after 27 epochs because no improvements over 10 subsequent epochs!
Epoch 27/100, Train Loss: 0.3863, Val Loss: 0.3883
The function took 70.48 seconds to execute.
No description has been provided for this image
Training Accuracy: 92.5998
Training Precision Scores: [31.20728929 98.33901193]
Training Recall Scores: [63.72093023 93.86178862]
Training F1 Scores: [41.89602446 96.04825291]

Validation Accuracy: 93.5358
Validation Precision Scores: [32.7014218  98.98175647]
Validation Recall Scores: [74.19354839 94.26262626]
Validation F1 Scores: [45.39473684 96.56456954]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4983, Val Loss: 0.4490
Epoch 40/100, Train Loss: 0.4622, Val Loss: 0.4087
Early stopping triggered after 44 epochs because no improvements over 10 subsequent epochs!
Epoch 44/100, Train Loss: 0.4802, Val Loss: 0.4559
The function took 131.50 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.6212
Training Precision Scores: [19.46959897 98.5213205 ]
Training Recall Scores: [70.         87.34756098]
Training F1 Scores: [30.46558704 92.59857789]

Validation Accuracy: 86.2150
Validation Precision Scores: [16.45244216 98.66911427]
Validation Recall Scores: [68.8172043  86.86868687]
Validation F1 Scores: [26.5560166  92.39363988]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5455, Val Loss: 0.4950
Epoch 40/100, Train Loss: 0.5364, Val Loss: 0.4822
Early stopping triggered after 52 epochs because no improvements over 10 subsequent epochs!
Epoch 52/100, Train Loss: 0.5328, Val Loss: 0.5277
The function took 186.26 seconds to execute.
No description has been provided for this image
Training Accuracy: 82.2687
Training Precision Scores: [14.49719245 98.24329202]
Training Recall Scores: [66.04651163 82.97764228]
Training F1 Scores: [23.77563834 89.9674949 ]

Validation Accuracy: 81.9315
Validation Precision Scores: [13.12127237 98.69249395]
Validation Recall Scores: [70.96774194 82.34343434]
Validation F1 Scores: [22.14765101 89.77973568]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5533, Val Loss: 0.5672
Early stopping triggered after 29 epochs because no improvements over 10 subsequent epochs!
Epoch 29/100, Train Loss: 0.5101, Val Loss: 0.4901
The function took 113.91 seconds to execute.
No description has been provided for this image
Training Accuracy: 79.9416
Training Precision Scores: [12.44239631 98.02469136]
Training Recall Scores: [62.79069767 80.69105691]
Training F1 Scores: [20.76923077 88.51727982]

Validation Accuracy: 80.3349
Validation Precision Scores: [11.99261993 98.61796644]
Validation Recall Scores: [69.89247312 80.72727273]
Validation F1 Scores: [20.47244094 88.78027105]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.6289, Val Loss: 0.6024
Early stopping triggered after 34 epochs because no improvements over 10 subsequent epochs!
Epoch 34/100, Train Loss: 0.5164, Val Loss: 0.4968
The function took 147.34 seconds to execute.
No description has been provided for this image
Training Accuracy: 80.6816
Training Precision Scores: [13.         98.07833537]
Training Recall Scores: [63.48837209 81.43292683]
Training F1 Scores: [21.58102767 88.98389783]

Validation Accuracy: 80.5685
Validation Precision Scores: [11.98501873 98.57423795]
Validation Recall Scores: [68.8172043  81.01010101]
Validation F1 Scores: [20.41467305 88.93324462]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5504, Val Loss: 0.4642
Early stopping triggered after 37 epochs because no improvements over 10 subsequent epochs!
Epoch 37/100, Train Loss: 0.5703, Val Loss: 0.4478
The function took 227.91 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.7089
Training Precision Scores: [17.55725191 97.99524295]
Training Recall Scores: [58.8372093  87.92682927]
Training F1 Scores: [27.0443613  92.68841395]

Validation Accuracy: 85.9034
Validation Precision Scores: [14.88250653 98.35240275]
Validation Recall Scores: [61.29032258 86.82828283]
Validation F1 Scores: [23.94957983 92.23175966]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3687, Val Loss: 0.3807
Epoch 40/100, Train Loss: 0.2608, Val Loss: 0.3019
Early stopping triggered after 55 epochs because no improvements over 10 subsequent epochs!
Epoch 55/100, Train Loss: 0.2327, Val Loss: 0.3513
The function took 141.09 seconds to execute.
No description has been provided for this image
Training Accuracy: 92.8530
Training Precision Scores: [36.15664845 99.64020933]
Training Recall Scores: [92.3255814  92.87601626]
Training F1 Scores: [51.96335079 96.13928045]

Validation Accuracy: 92.0950
Validation Precision Scores: [29.16666667 99.30555556]
Validation Recall Scores: [82.79569892 92.44444444]
Validation F1 Scores: [43.1372549  95.75224942]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.2930, Val Loss: 0.3499
Early stopping triggered after 34 epochs because no improvements over 10 subsequent epochs!
Epoch 34/100, Train Loss: 0.2475, Val Loss: 0.3510
The function took 87.09 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.2960
Training Precision Scores: [25.22464698 99.57529844]
Training Recall Scores: [91.39534884 88.16056911]
Training F1 Scores: [39.53722334 93.52091419]

Validation Accuracy: 87.8505
Validation Precision Scores: [20.4851752  99.22621757]
Validation Recall Scores: [81.72043011 88.08080808]
Validation F1 Scores: [32.75862069 93.32191781]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3650, Val Loss: 0.4621
Epoch 40/100, Train Loss: 0.2754, Val Loss: 0.3950
Early stopping triggered after 42 epochs because no improvements over 10 subsequent epochs!
Epoch 42/100, Train Loss: 0.2623, Val Loss: 0.3861
The function took 107.70 seconds to execute.
No description has been provided for this image
Training Accuracy: 87.1373
Training Precision Scores: [22.61831592 99.28265649]
Training Recall Scores: [85.58139535 87.20528455]
Training F1 Scores: [35.78026252 92.85289185]

Validation Accuracy: 86.8380
Validation Precision Scores: [18.98734177 99.17165209]
Validation Recall Scores: [80.64516129 87.07070707]
Validation F1 Scores: [30.73770492 92.72805508]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4698, Val Loss: 0.4499
Early stopping triggered after 24 epochs because no improvements over 10 subsequent epochs!
Epoch 24/100, Train Loss: 0.4268, Val Loss: 0.4094
The function took 71.60 seconds to execute.
No description has been provided for this image
Training Accuracy: 84.8880
Training Precision Scores: [19.0397351  98.99503429]
Training Recall Scores: [80.23255814 85.09146341]
Training F1 Scores: [30.77609277 91.51819871]

Validation Accuracy: 84.7352
Validation Precision Scores: [16.40449438 99.05793688]
Validation Recall Scores: [78.49462366 84.96969697]
Validation F1 Scores: [27.13754647 91.47455415]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5004, Val Loss: 0.4955
Early stopping triggered after 21 epochs because no improvements over 10 subsequent epochs!
Epoch 21/100, Train Loss: 0.5102, Val Loss: 0.4821
The function took 75.29 seconds to execute.
No description has been provided for this image
Training Accuracy: 81.3145
Training Precision Scores: [14.5644931  98.48206635]
Training Recall Scores: [71.1627907  81.75813008]
Training F1 Scores: [24.18016594 89.34421678]

Validation Accuracy: 81.8925
Validation Precision Scores: [14.23076923 99.07226562]
Validation Recall Scores: [79.56989247 81.97979798]
Validation F1 Scores: [24.14355628 89.71921291]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5312, Val Loss: 0.4531
Early stopping triggered after 30 epochs because no improvements over 10 subsequent epochs!
Epoch 30/100, Train Loss: 0.5243, Val Loss: 0.5038
The function took 117.84 seconds to execute.
No description has been provided for this image
Training Accuracy: 82.9017
Training Precision Scores: [14.31646932 98.05040418]
Training Recall Scores: [61.86046512 83.82113821]
Training F1 Scores: [23.25174825 90.37913653]

Validation Accuracy: 82.7103
Validation Precision Scores: [13.3611691  98.61177597]
Validation Recall Scores: [68.8172043  83.23232323]
Validation F1 Scores: [22.37762238 90.2716915 ]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5500, Val Loss: 0.4582
Epoch 40/100, Train Loss: 0.5256, Val Loss: 0.4514
Early stopping triggered after 45 epochs because no improvements over 10 subsequent epochs!
Epoch 45/100, Train Loss: 0.4806, Val Loss: 0.4778
The function took 194.97 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.7575
Training Precision Scores: [18.01925722 98.09437387]
Training Recall Scores: [60.93023256 87.88617886]
Training F1 Scores: [27.81316348 92.71012007]

Validation Accuracy: 87.1885
Validation Precision Scores: [17.93478261 98.77272727]
Validation Recall Scores: [70.96774194 87.7979798 ]
Validation F1 Scores: [28.63340564 92.96256684]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5956, Val Loss: 0.6739
Early stopping triggered after 38 epochs because no improvements over 10 subsequent epochs!
Epoch 38/100, Train Loss: 0.5701, Val Loss: 0.5091
The function took 235.43 seconds to execute.
No description has been provided for this image
Training Accuracy: 81.7137
Training Precision Scores: [14.71734893 98.44244342]
Training Recall Scores: [70.23255814 82.21544715]
Training F1 Scores: [24.33521354 89.60017721]

Validation Accuracy: 82.5545
Validation Precision Scores: [14.42885772 98.98501692]
Validation Recall Scores: [77.41935484 82.74747475]
Validation F1 Scores: [24.32432432 90.14084507]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.2625, Val Loss: 0.3526
Early stopping triggered after 32 epochs because no improvements over 10 subsequent epochs!
Epoch 32/100, Train Loss: 0.1860, Val Loss: 0.4197
The function took 81.94 seconds to execute.
No description has been provided for this image
Training Accuracy: 87.8968
Training Precision Scores: [25.16799023 99.79149774]
Training Recall Scores: [95.81395349 87.55081301]
Training F1 Scores: [39.86453798 93.27126076]

Validation Accuracy: 87.3053
Validation Precision Scores: [19.42257218 99.13123   ]
Validation Recall Scores: [79.56989247 87.5959596 ]
Validation F1 Scores: [31.22362869 93.00729301]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 18 epochs because no improvements over 10 subsequent epochs!
Epoch 18/100, Train Loss: 0.3496, Val Loss: 0.3778
The function took 46.06 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.6952
Training Precision Scores: [23.87419585 98.91782212]
Training Recall Scores: [77.6744186  89.17682927]
Training F1 Scores: [36.52268999 93.7950938 ]

Validation Accuracy: 89.3302
Validation Precision Scores: [21.63009404 98.93285905]
Validation Recall Scores: [74.19354839 89.8989899 ]
Validation F1 Scores: [33.49514563 94.19983065]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4401, Val Loss: 0.3865
Epoch 40/100, Train Loss: 0.3645, Val Loss: 0.3602
Early stopping triggered after 57 epochs because no improvements over 10 subsequent epochs!
Epoch 57/100, Train Loss: 0.2987, Val Loss: 0.3767
The function took 170.23 seconds to execute.
No description has been provided for this image
Training Accuracy: 87.5949
Training Precision Scores: [22.73901809 99.1057097 ]
Training Recall Scores: [81.86046512 87.84552846]
Training F1 Scores: [35.59150657 93.13651546]

Validation Accuracy: 87.2664
Validation Precision Scores: [19.04761905 99.04109589]
Validation Recall Scores: [77.41935484 87.63636364]
Validation F1 Scores: [30.57324841 92.9903537 ]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4564, Val Loss: 0.4170
Epoch 40/100, Train Loss: 0.4232, Val Loss: 0.3851
Early stopping triggered after 59 epochs because no improvements over 10 subsequent epochs!
Epoch 59/100, Train Loss: 0.4098, Val Loss: 0.4285
The function took 211.90 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.3486
Training Precision Scores: [20.25703794 98.85363594]
Training Recall Scores: [76.97674419 86.75813008]
Training F1 Scores: [32.07364341 92.41177744]

Validation Accuracy: 85.2804
Validation Precision Scores: [16.31205674 98.88111888]
Validation Recall Scores: [74.19354839 85.6969697 ]
Validation F1 Scores: [26.74418605 91.81818182]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4749, Val Loss: 0.4585
Early stopping triggered after 27 epochs because no improvements over 10 subsequent epochs!
Epoch 27/100, Train Loss: 0.4763, Val Loss: 0.4706
The function took 106.36 seconds to execute.
No description has been provided for this image
Training Accuracy: 82.6485
Training Precision Scores: [15.29774127 98.41384283]
Training Recall Scores: [69.30232558 83.23170732]
Training F1 Scores: [25.06307822 90.18830525]

Validation Accuracy: 82.7492
Validation Precision Scores: [13.69294606 98.70565676]
Validation Recall Scores: [70.96774194 83.19191919]
Validation F1 Scores: [22.95652174 90.28721772]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5002, Val Loss: 0.4705
Epoch 40/100, Train Loss: 0.4820, Val Loss: 0.4745
Early stopping triggered after 43 epochs because no improvements over 10 subsequent epochs!
Epoch 43/100, Train Loss: 0.4835, Val Loss: 0.4752
The function took 187.48 seconds to execute.
No description has been provided for this image
Training Accuracy: 87.9455
Training Precision Scores: [19.94047619 98.1850773 ]
Training Recall Scores: [62.3255814  89.06504065]
Training F1 Scores: [30.21420519 93.40296281]

Validation Accuracy: 88.8240
Validation Precision Scores: [19.10828025 98.53593611]
Validation Recall Scores: [64.51612903 89.73737374]
Validation F1 Scores: [29.48402948 93.93106365]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5148, Val Loss: 0.4827
Early stopping triggered after 22 epochs because no improvements over 10 subsequent epochs!
Epoch 22/100, Train Loss: 0.5023, Val Loss: 0.5167
The function took 136.97 seconds to execute.
No description has been provided for this image
Training Accuracy: 80.3311
Training Precision Scores: [13.53211009 98.33127318]
Training Recall Scores: [68.60465116 80.84349593]
Training F1 Scores: [22.60536398 88.73396542]

Validation Accuracy: 80.0234
Validation Precision Scores: [11.67883212 98.56435644]
Validation Recall Scores: [68.8172043  80.44444444]
Validation F1 Scores: [19.96879875 88.58731924]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.2019, Val Loss: 0.3709
Early stopping triggered after 28 epochs because no improvements over 10 subsequent epochs!
Epoch 28/100, Train Loss: 0.1565, Val Loss: 0.4636
The function took 71.79 seconds to execute.
No description has been provided for this image
Training Accuracy: 91.6845
Training Precision Scores: [32.87560582 99.74534987]
Training Recall Scores: [94.65116279 91.55487805]
Training F1 Scores: [48.80095923 95.47477745]

Validation Accuracy: 90.3816
Validation Precision Scores: [24.67105263 99.204947  ]
Validation Recall Scores: [80.64516129 90.74747475]
Validation F1 Scores: [37.78337531 94.78792994]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3084, Val Loss: 0.3381
Early stopping triggered after 30 epochs because no improvements over 10 subsequent epochs!
Epoch 30/100, Train Loss: 0.2975, Val Loss: 0.3764
The function took 90.13 seconds to execute.
No description has been provided for this image
Training Accuracy: 89.1529
Training Precision Scores: [26.73469388 99.57954545]
Training Recall Scores: [91.39534884 89.05487805]
Training F1 Scores: [41.36842105 94.02360515]

Validation Accuracy: 88.2399
Validation Precision Scores: [20.56338028 99.09624944]
Validation Recall Scores: [78.49462366 88.60606061]
Validation F1 Scores: [32.58928571 93.55802048]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4369, Val Loss: 0.4589
Early stopping triggered after 31 epochs because no improvements over 10 subsequent epochs!
Epoch 31/100, Train Loss: 0.4092, Val Loss: 0.4276
The function took 111.52 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.0370
Training Precision Scores: [19.53883495 98.7473904 ]
Training Recall Scores: [74.88372093 86.52439024]
Training F1 Scores: [30.99133782 92.23269418]

Validation Accuracy: 86.2928
Validation Precision Scores: [17.38035264 98.89451865]
Validation Recall Scores: [74.19354839 86.74747475]
Validation F1 Scores: [28.16326531 92.42359019]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4658, Val Loss: 0.4689
Early stopping triggered after 35 epochs because no improvements over 10 subsequent epochs!
Epoch 35/100, Train Loss: 0.4000, Val Loss: 0.4599
The function took 138.34 seconds to execute.
No description has been provided for this image
Training Accuracy: 84.8393
Training Precision Scores: [18.78116343 98.92498523]
Training Recall Scores: [78.8372093  85.10162602]
Training F1 Scores: [30.33557047 91.49412729]

Validation Accuracy: 83.0997
Validation Precision Scores: [14.98973306 99.03892359]
Validation Recall Scores: [78.49462366 83.27272727]
Validation F1 Scores: [25.17241379 90.47410009]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4639, Val Loss: 0.4308
Early stopping triggered after 27 epochs because no improvements over 10 subsequent epochs!
Epoch 27/100, Train Loss: 0.4502, Val Loss: 0.4576
The function took 117.83 seconds to execute.
No description has been provided for this image
Training Accuracy: 78.3252
Training Precision Scores: [13.67313916 98.82021031]
Training Recall Scores: [78.60465116 78.31300813]
Training F1 Scores: [23.29427981 87.37952149]

Validation Accuracy: 78.6215
Validation Precision Scores: [12.12624585 98.982706  ]
Validation Recall Scores: [78.49462366 78.62626263]
Validation F1 Scores: [21.00719424 87.63791939]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4782, Val Loss: 0.4573
Early stopping triggered after 23 epochs because no improvements over 10 subsequent epochs!
Epoch 23/100, Train Loss: 0.4977, Val Loss: 0.4395
The function took 142.85 seconds to execute.
No description has been provided for this image
Training Accuracy: 87.4489
Training Precision Scores: [19.07847372 98.14210111]
Training Recall Scores: [61.62790698 88.57723577]
Training F1 Scores: [29.1368884  93.11468404]

Validation Accuracy: 87.8115
Validation Precision Scores: [18.57142857 98.73760144]
Validation Recall Scores: [69.89247312 88.48484848]
Validation F1 Scores: [29.34537246 93.33049222]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.2777, Val Loss: 0.3811
Early stopping triggered after 26 epochs because no improvements over 10 subsequent epochs!
Epoch 26/100, Train Loss: 0.2391, Val Loss: 0.4375
The function took 77.99 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.6271
Training Precision Scores: [25.72368421 99.55428571]
Training Recall Scores: [90.93023256 88.52642276]
Training F1 Scores: [40.1025641  93.71705218]

Validation Accuracy: 87.3832
Validation Precision Scores: [19.36339523 99.08717481]
Validation Recall Scores: [78.49462366 87.71717172]
Validation F1 Scores: [31.06382979 93.05615088]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 15 epochs because no improvements over 10 subsequent epochs!
Epoch 15/100, Train Loss: 0.4280, Val Loss: 0.4605
The function took 54.25 seconds to execute.
No description has been provided for this image
Training Accuracy: 90.8374
Training Precision Scores: [26.7938238  98.52764751]
Training Recall Scores: [68.60465116 91.80894309]
Training F1 Scores: [38.53690398 95.04971329]

Validation Accuracy: 91.2383
Validation Precision Scores: [25.55555556 98.95561358]
Validation Recall Scores: [74.19354839 91.87878788]
Validation F1 Scores: [38.01652893 95.28598366]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4494, Val Loss: 0.4666
Early stopping triggered after 39 epochs because no improvements over 10 subsequent epochs!
Epoch 39/100, Train Loss: 0.4450, Val Loss: 0.4094
The function took 155.88 seconds to execute.
No description has been provided for this image
Training Accuracy: 84.0506
Training Precision Scores: [18.96197328 99.26717924]
Training Recall Scores: [85.81395349 83.97357724]
Training F1 Scores: [31.06060606 90.98216252]

Validation Accuracy: 82.7492
Validation Precision Scores: [14.85943775 99.0821256 ]
Validation Recall Scores: [79.56989247 82.86868687]
Validation F1 Scores: [25.04230118 90.2530253 ]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4277, Val Loss: 0.4412
Early stopping triggered after 28 epochs because no improvements over 10 subsequent epochs!
Epoch 28/100, Train Loss: 0.3929, Val Loss: 0.4036
The function took 123.58 seconds to execute.
No description has been provided for this image
Training Accuracy: 90.5940
Training Precision Scores: [26.24113475 98.53423758]
Training Recall Scores: [68.8372093  91.54471545]
Training F1 Scores: [37.99743261 94.91096829]

Validation Accuracy: 91.0826
Validation Precision Scores: [24.62686567 98.82608696]
Validation Recall Scores: [70.96774194 91.83838384]
Validation F1 Scores: [36.56509695 95.20418848]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 17 epochs because no improvements over 10 subsequent epochs!
Epoch 17/100, Train Loss: 0.4826, Val Loss: 0.4524
The function took 106.47 seconds to execute.
No description has been provided for this image
Training Accuracy: 82.5901
Training Precision Scores: [15.49796748 98.49433871]
Training Recall Scores: [70.93023256 83.0995935 ]
Training F1 Scores: [25.43786489 90.14441627]

Validation Accuracy: 83.0607
Validation Precision Scores: [14.375      98.85057471]
Validation Recall Scores: [74.19354839 83.39393939]
Validation F1 Scores: [24.08376963 90.46679816]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 15 epochs because no improvements over 10 subsequent epochs!
Epoch 15/100, Train Loss: 0.1825, Val Loss: 0.4723
The function took 57.72 seconds to execute.
No description has been provided for this image
Training Accuracy: 89.0847
Training Precision Scores: [24.65150404 98.94465028]
Training Recall Scores: [78.13953488 89.56300813]
Training F1 Scores: [37.47908533 94.02037659]

Validation Accuracy: 88.6682
Validation Precision Scores: [20.35928144 98.88093107]
Validation Recall Scores: [73.11827957 89.25252525]
Validation F1 Scores: [31.8501171  93.82034402]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 15 epochs because no improvements over 10 subsequent epochs!
Epoch 15/100, Train Loss: 0.3965, Val Loss: 0.4454
The function took 62.05 seconds to execute.
No description has been provided for this image
Training Accuracy: 83.1061
Training Precision Scores: [17.29323308 98.97280967]
Training Recall Scores: [80.23255814 83.23170732]
Training F1 Scores: [28.45360825 90.42230196]

Validation Accuracy: 83.0218
Validation Precision Scores: [15.0712831  99.08521907]
Validation Recall Scores: [79.56989247 83.15151515]
Validation F1 Scores: [25.34246575 90.42179262]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4046, Val Loss: 0.4632
Early stopping triggered after 27 epochs because no improvements over 10 subsequent epochs!
Epoch 27/100, Train Loss: 0.4374, Val Loss: 0.4332
The function took 121.41 seconds to execute.
No description has been provided for this image
Training Accuracy: 82.6095
Training Precision Scores: [16.99123661 99.01411879]
Training Recall Scores: [81.1627907  82.67276423]
Training F1 Scores: [28.09983897 90.10855117]

Validation Accuracy: 83.2165
Validation Precision Scores: [14.93775934 98.99328859]
Validation Recall Scores: [77.41935484 83.43434343]
Validation F1 Scores: [25.04347826 90.55031791]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4142, Val Loss: 0.4088
Early stopping triggered after 22 epochs because no improvements over 10 subsequent epochs!
Epoch 22/100, Train Loss: 0.4625, Val Loss: 0.4649
The function took 136.59 seconds to execute.
No description has been provided for this image
Training Accuracy: 78.0234
Training Precision Scores: [14.02126822 99.04281464]
Training Recall Scores: [82.79069767 77.81504065]
Training F1 Scores: [23.98113843 87.15497126]

Validation Accuracy: 77.7648
Validation Precision Scores: [12.06349206 99.12280702]
Validation Recall Scores: [81.72043011 77.61616162]
Validation F1 Scores: [21.02351314 87.06095627]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 17 epochs because no improvements over 10 subsequent epochs!
Epoch 17/100, Train Loss: 0.2423, Val Loss: 0.5318
The function took 72.44 seconds to execute.
No description has been provided for this image
Training Accuracy: 90.0195
Training Precision Scores: [27.37642586 99.21831379]
Training Recall Scores: [83.72093023 90.29471545]
Training F1 Scores: [41.26074499 94.54642192]

Validation Accuracy: 88.5514
Validation Precision Scores: [20.17804154 98.87942627]
Validation Recall Scores: [73.11827957 89.13131313]
Validation F1 Scores: [31.62790698 93.75265618]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3648, Val Loss: 0.4749
Early stopping triggered after 21 epochs because no improvements over 10 subsequent epochs!
Epoch 21/100, Train Loss: 0.3530, Val Loss: 0.4558
The function took 96.76 seconds to execute.
No description has been provided for this image
Training Accuracy: 83.2911
Training Precision Scores: [17.85       99.11729141]
Training Recall Scores: [83.02325581 83.30284553]
Training F1 Scores: [29.38271605 90.52457206]

Validation Accuracy: 82.5935
Validation Precision Scores: [14.17004049 98.89103182]
Validation Recall Scores: [75.2688172  82.86868687]
Validation F1 Scores: [23.85008518 90.17366454]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 64
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 17 epochs because no improvements over 10 subsequent epochs!
Epoch 17/100, Train Loss: 0.4210, Val Loss: 0.4302
The function took 108.43 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.3252
Training Precision Scores: [21.37006701 98.39811807]
Training Recall Scores: [66.74418605 89.26829268]
Training F1 Scores: [32.37450649 93.61112591]

Validation Accuracy: 88.0452
Validation Precision Scores: [19.94382022 99.00542495]
Validation Recall Scores: [76.34408602 88.48484848]
Validation F1 Scores: [31.62583519 93.449968  ]

In [15]:
# 16 64

train_dataset = TimeSeriesDataset(X_train, y_train)
val_dataset = TimeSeriesDataset(X_val, y_val)
test_dataset = TimeSeriesDataset(X_test, y_test)

train_loader = DataLoader(train_dataset, batch_size=64, collate_fn=collate_fn,shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)

embedding_size=16
hidden_dim=64

vocabulary_size = len(vectorizer.vocabulary_) + 2  # Add 2 for out-of-vocabulary and padding
output_size = 2

model = BiDirectionalRNN(vocabulary_size, embedding_size, hidden_dim, output_size)
model = model.to(device)

class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)  # Computing class weights
class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)

optimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.01)
num_epochs = 100
training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs)

test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model(model,test_loader,device)
stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.2830, Val Loss: 0.3820
Early stopping triggered after 22 epochs because no improvements over 10 subsequent epochs!
Epoch 22/100, Train Loss: 0.2298, Val Loss: 0.3709
The function took 57.16 seconds to execute.
No description has been provided for this image
Test Accuracy: 88.1344
Test Precision Scores: [19.32938856 99.14759274]
Test Recall Scores: [78.4        88.47725032]
Test F1 Scores: [31.01265823 93.509007  ]

Out[15]:
<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7a939d2d6050>
No description has been provided for this image
No description has been provided for this image

LSTM¶

In [16]:
class LSTMClassifier(nn.Module):
    def __init__(self, vocab_size, embed_size, hid_size, out_size):
        super(LSTMClassifier, self).__init__()

        # 1) embedding layer: vocab_size × d
        self.embedding = nn.Embedding(vocab_size, embed_size, padding_idx=max(vectorizer.vocabulary_.values()) + 1)

        # 2) first linear layer: (seq_len·d) → H
        #batch first is used to specify that the matrix in input has dimension:
        #(batch_size, sequence_length, input_size)
        self.lstm = nn.LSTM(input_size=embed_size, hidden_size=hid_size, batch_first=True, num_layers=2, bidirectional=True, dropout=0.5)

        # 3) second linear layer: H → 2
        self.linear = nn.Linear(hid_size * 2, out_size)

        self.dropout = nn.Dropout(0.5)

    """
    Args:
        - input_seq: The input sequence tensor of shape
         (batch_size, sequence_length, input_size).
     Output of rnn:
         - **rnn_out** contains the output for all time steps.
           It has shape (batch_size, sequence_length, hidden_size).
         - **h** contains the hidden states for all layers at the last time step.
            It has shape (num_layers, batch_size, hidden_size).
         - if different time-series have different sequence_length, rnn_out can generate errors
         - with h we get batch_size instead of sequence_length
           ==> no problem with different sequence_length

         - We do not return them in the tracking loop to
             - avoid problems with the gradient vanish/exploding
             - use the windows independently (if possible) and shuffle the data
        Returns:
         - predictions: The predicted pollution levels for the input sequence.
         Use h from last hidden node helps with time-series having different lengths
    """
    def forward(self, x: torch.LongTensor, lengths):

        # embedding lookup → (B, seq_len, d) # certo, perchè se prima avevo per ogni parola d dim, se qui ho 100 parole avrò 100xd dimensioni
        x = self.embedding(x.long())
        x = self.dropout(x)
        x = pack_padded_sequence(x, lengths.cpu(), batch_first=True, enforce_sorted=True)

        x, (hidden, cell) = self.lstm(x)

        # Concatenate the final forward and backward hidden states
        last_forward = hidden[-2, :, :]  # Last layer's forward direction
        last_backward = hidden[-1, :, :]  # Last layer's backward direction

        # Concatenate the final forward and backward hidden states
        hidden_cat = torch.cat((last_forward, last_backward), dim=1)

        output = self.linear(hidden_cat)

        return output
In [26]:
train_dataset = TimeSeriesDataset(X_train, y_train)
val_dataset = TimeSeriesDataset(X_val, y_val)
test_dataset = TimeSeriesDataset(X_test, y_test)

train_loader = DataLoader(train_dataset, batch_size=64, collate_fn=collate_fn,shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)

embedding_sizes = [4,8,16,32,64,128,256]
hidden_dims = [8,16,32,64,128,256,320,384,512]

for i in range(len(embedding_sizes)):
    for j in range(i,len(hidden_dims)):
        embedding_size=embedding_sizes[i]
        hidden_size=hidden_dims[j]
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print(f"Trying with embedding size {embedding_size} and hidden dim {hidden_dim}")

        vocabulary_size = len(vectorizer.vocabulary_) + 2  # Add 2 for out-of-vocabulary and padding
        output_size = 2

        model = LSTMClassifier(vocabulary_size, embedding_size, hidden_size, output_size)
        model = model.to(device)
        
        class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)  # Computing class weights
        class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
        criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
        
        optimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.01)
        num_epochs = 100
        training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs)
        
        train_accuracy, train_precisionscores, train_recallscores, train_f1scores, _, _ = testing_model(model,train_loader,device)
        val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model(model,val_loader,device)
        
        stats(train_accuracy, train_precisionscores, train_recallscores, train_f1scores, 'Training')
        stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4848, Val Loss: 0.4753
Early stopping triggered after 32 epochs because no improvements over 10 subsequent epochs!
Epoch 32/100, Train Loss: 0.4369, Val Loss: 0.4301
The function took 90.38 seconds to execute.
No description has been provided for this image
Training Accuracy: 78.8121
Training Precision Scores: [14.01483924 98.85262621]
Training Recall Scores: [79.06976744 78.80081301]
Training F1 Scores: [23.80952381 87.69509161]

Validation Accuracy: 79.5561
Validation Precision Scores: [13.13993174 99.19273461]
Validation Recall Scores: [82.79569892 79.43434343]
Validation F1 Scores: [22.68041237 88.22077631]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4722, Val Loss: 0.4299
Epoch 40/100, Train Loss: 0.3970, Val Loss: 0.3658
Early stopping triggered after 43 epochs because no improvements over 10 subsequent epochs!
Epoch 43/100, Train Loss: 0.4133, Val Loss: 0.3598
The function took 121.20 seconds to execute.
No description has been provided for this image
Training Accuracy: 84.7712
Training Precision Scores: [19.45043103 99.1799382 ]
Training Recall Scores: [83.95348837 84.80691057]
Training F1 Scores: [31.58355206 91.4320149 ]

Validation Accuracy: 84.8131
Validation Precision Scores: [17.07317073 99.24421351]
Validation Recall Scores: [82.79569892 84.88888889]
Validation F1 Scores: [28.30882353 91.50696864]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4546, Val Loss: 0.3881
Epoch 40/100, Train Loss: 0.4080, Val Loss: 0.3317
Early stopping triggered after 59 epochs because no improvements over 10 subsequent epochs!
Epoch 59/100, Train Loss: 0.3264, Val Loss: 0.3321
The function took 174.52 seconds to execute.
No description has been provided for this image
Training Accuracy: 89.8734
Training Precision Scores: [26.71755725 99.10714286]
Training Recall Scores: [81.39534884 90.24390244]
Training F1 Scores: [40.22988506 94.46808511]

Validation Accuracy: 89.6028
Validation Precision Scores: [23.31288344 99.24174844]
Validation Recall Scores: [81.72043011 89.8989899 ]
Validation F1 Scores: [36.27684964 94.33962264]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4446, Val Loss: 0.3676
Epoch 40/100, Train Loss: 0.3656, Val Loss: 0.3144
Epoch 60/100, Train Loss: 0.3176, Val Loss: 0.2901
Early stopping triggered after 73 epochs because no improvements over 10 subsequent epochs!
Epoch 73/100, Train Loss: 0.3019, Val Loss: 0.2807
The function took 219.96 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.3252
Training Precision Scores: [25.6799494 99.7237887]
Training Recall Scores: [94.41860465 88.05894309]
Training F1 Scores: [40.37792143 93.52906255]

Validation Accuracy: 88.3956
Validation Precision Scores: [21.7630854  99.36507937]
Validation Recall Scores: [84.94623656 88.52525253]
Validation F1 Scores: [34.64912281 93.63247863]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4343, Val Loss: 0.3774
Epoch 40/100, Train Loss: 0.3284, Val Loss: 0.3239
Early stopping triggered after 57 epochs because no improvements over 10 subsequent epochs!
Epoch 57/100, Train Loss: 0.2932, Val Loss: 0.3372
The function took 225.70 seconds to execute.
No description has been provided for this image
Training Accuracy: 92.7069
Training Precision Scores: [34.99529633 99.37004453]
Training Recall Scores: [86.51162791 92.97764228]
Training F1 Scores: [49.83255191 96.0676222 ]

Validation Accuracy: 92.5623
Validation Precision Scores: [30.24193548 99.22413793]
Validation Recall Scores: [80.64516129 93.01010101]
Validation F1 Scores: [43.98826979 96.01668405]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4751, Val Loss: 0.3806
Epoch 40/100, Train Loss: 0.3945, Val Loss: 0.3470
Early stopping triggered after 47 epochs because no improvements over 10 subsequent epochs!
Epoch 47/100, Train Loss: 0.3432, Val Loss: 0.3495
The function took 323.10 seconds to execute.
No description has been provided for this image
Training Accuracy: 85.4722
Training Precision Scores: [20.5654102  99.30309473]
Training Recall Scores: [86.27906977 85.43699187]
Training F1 Scores: [33.21396598 91.84966678]

Validation Accuracy: 85.8645
Validation Precision Scores: [18.30985915 99.29971989]
Validation Recall Scores: [83.87096774 85.93939394]
Validation F1 Scores: [30.05780347 92.13775179]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5626, Val Loss: 0.4673
Epoch 40/100, Train Loss: 0.5162, Val Loss: 0.4244
Epoch 60/100, Train Loss: 0.4560, Val Loss: 0.4116
Epoch 80/100, Train Loss: 0.3689, Val Loss: 0.3630
Early stopping triggered after 96 epochs because no improvements over 10 subsequent epochs!
Epoch 96/100, Train Loss: 0.3140, Val Loss: 0.3561
The function took 770.88 seconds to execute.
No description has been provided for this image
Training Accuracy: 82.9211
Training Precision Scores: [18.74409821 99.59519136]
Training Recall Scores: [92.3255814 82.5101626]
Training F1 Scores: [31.16169545 90.25122277]

Validation Accuracy: 82.9050
Validation Precision Scores: [15.4        99.22630561]
Validation Recall Scores: [82.79569892 82.90909091]
Validation F1 Scores: [25.96964587 90.33678186]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4866, Val Loss: 0.3846
Epoch 40/100, Train Loss: 0.3645, Val Loss: 0.3096
Early stopping triggered after 55 epochs because no improvements over 10 subsequent epochs!
Epoch 55/100, Train Loss: 0.3079, Val Loss: 0.3259
The function took 512.56 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.6173
Training Precision Scores: [25.31730127 99.41867092]
Training Recall Scores: [88.13953488 88.63821138]
Training F1 Scores: [39.33575506 93.7194434 ]

Validation Accuracy: 88.1231
Validation Precision Scores: [21.03825137 99.27338783]
Validation Recall Scores: [82.79569892 88.32323232]
Validation F1 Scores: [33.55119826 93.47872568]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 4 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5427, Val Loss: 0.4812
Epoch 40/100, Train Loss: 0.5127, Val Loss: 0.4427
Epoch 60/100, Train Loss: 0.4260, Val Loss: 0.3632
Epoch 80/100, Train Loss: 0.3437, Val Loss: 0.2973
Epoch 100/100, Train Loss: 0.2592, Val Loss: 0.2644
The function took 1295.55 seconds to execute.
No description has been provided for this image
Training Accuracy: 89.6203
Training Precision Scores: [28.03867403 99.72795285]
Training Recall Scores: [94.41860465 89.41056911]
Training F1 Scores: [43.23748669 94.28785768]

Validation Accuracy: 89.3302
Validation Precision Scores: [23.61516035 99.46067416]
Validation Recall Scores: [87.09677419 89.41414141]
Validation F1 Scores: [37.1559633  94.17021277]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3968, Val Loss: 0.3498
Epoch 40/100, Train Loss: 0.3312, Val Loss: 0.2744
Early stopping triggered after 51 epochs because no improvements over 10 subsequent epochs!
Epoch 51/100, Train Loss: 0.3015, Val Loss: 0.2640
The function took 144.85 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.5531
Training Precision Scores: [22.27405248 99.43892461]
Training Recall Scores: [88.8372093  86.45325203]
Training F1 Scores: [35.61771562 92.49252514]

Validation Accuracy: 86.6822
Validation Precision Scores: [20.14388489 99.58158996]
Validation Recall Scores: [90.32258065 86.54545455]
Validation F1 Scores: [32.94117647 92.60700389]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3772, Val Loss: 0.3162
Epoch 40/100, Train Loss: 0.3012, Val Loss: 0.3339
Epoch 60/100, Train Loss: 0.2327, Val Loss: 0.2085
Epoch 80/100, Train Loss: 0.2110, Val Loss: 0.2154
Early stopping triggered after 94 epochs because no improvements over 10 subsequent epochs!
Epoch 94/100, Train Loss: 0.1928, Val Loss: 0.2086
The function took 278.05 seconds to execute.
No description has been provided for this image
Training Accuracy: 92.9893
Training Precision Scores: [37.16814159 99.89059081]
Training Recall Scores: [97.6744186  92.78455285]
Training F1 Scores: [53.84615385 96.20653319]

Validation Accuracy: 92.6791
Validation Precision Scores: [32.20973783 99.69578444]
Validation Recall Scores: [92.47311828 92.68686869]
Validation F1 Scores: [47.77777778 96.06365159]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3685, Val Loss: 0.3253
Epoch 40/100, Train Loss: 0.2873, Val Loss: 0.2614
Epoch 60/100, Train Loss: 0.2145, Val Loss: 0.2305
Early stopping triggered after 68 epochs because no improvements over 10 subsequent epochs!
Epoch 68/100, Train Loss: 0.2091, Val Loss: 0.2361
The function took 204.53 seconds to execute.
No description has been provided for this image
Training Accuracy: 93.7488
Training Precision Scores: [39.66861598 99.75118996]
Training Recall Scores: [94.65116279 93.70934959]
Training F1 Scores: [55.90659341 96.63592538]

Validation Accuracy: 93.5748
Validation Precision Scores: [35.36585366 99.74160207]
Validation Recall Scores: [93.5483871  93.57575758]
Validation F1 Scores: [51.32743363 96.56035022]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3799, Val Loss: 0.3259
Epoch 40/100, Train Loss: 0.2324, Val Loss: 0.2484
Early stopping triggered after 57 epochs because no improvements over 10 subsequent epochs!
Epoch 57/100, Train Loss: 0.1693, Val Loss: 0.2157
The function took 225.08 seconds to execute.
No description has been provided for this image
Training Accuracy: 95.1996
Training Precision Scores: [46.48044693 99.85066667]
Training Recall Scores: [96.74418605 95.13211382]
Training F1 Scores: [62.79245283 97.43429612]

Validation Accuracy: 94.0421
Validation Precision Scores: [36.60714286 99.53071672]
Validation Recall Scores: [88.17204301 94.26262626]
Validation F1 Scores: [51.73501577 96.82506744]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5091, Val Loss: 0.4553
Early stopping triggered after 25 epochs because no improvements over 10 subsequent epochs!
Epoch 25/100, Train Loss: 0.4523, Val Loss: 0.4013
The function took 171.61 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.0136
Training Precision Scores: [22.12943633 98.73202762]
Training Recall Scores: [73.95348837 88.62804878]
Training F1 Scores: [34.06534547 93.40759385]

Validation Accuracy: 88.5514
Validation Precision Scores: [20.52785924 98.96722048]
Validation Recall Scores: [75.2688172  89.05050505]
Validation F1 Scores: [32.25806452 93.74734156]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5027, Val Loss: 0.4288
Epoch 40/100, Train Loss: 0.4555, Val Loss: 0.3999
Epoch 60/100, Train Loss: 0.4093, Val Loss: 0.3962
Epoch 80/100, Train Loss: 0.2993, Val Loss: 0.3113
Epoch 100/100, Train Loss: 0.2190, Val Loss: 0.2351
The function took 803.30 seconds to execute.
No description has been provided for this image
Training Accuracy: 92.0351
Training Precision Scores: [34.12438625 99.85632184]
Training Recall Scores: [96.97674419 91.81910569]
Training F1 Scores: [50.4842615  95.66920796]

Validation Accuracy: 91.1215
Validation Precision Scores: [27.57475083 99.5588884 ]
Validation Recall Scores: [89.24731183 91.19191919]
Validation F1 Scores: [42.1319797  95.19190215]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 16 epochs because no improvements over 10 subsequent epochs!
Epoch 16/100, Train Loss: 0.5087, Val Loss: 0.4414
The function took 149.36 seconds to execute.
No description has been provided for this image
Training Accuracy: 82.7459
Training Precision Scores: [16.00810537 98.62584378]
Training Recall Scores: [73.48837209 83.1504065 ]
Training F1 Scores: [26.28951747 90.22937803]

Validation Accuracy: 82.7492
Validation Precision Scores: [14.71774194 99.03474903]
Validation Recall Scores: [78.49462366 82.90909091]
Validation F1 Scores: [24.78777589 90.25731251]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.5087, Val Loss: 0.4355
Epoch 40/100, Train Loss: 0.4794, Val Loss: 0.4131
Early stopping triggered after 52 epochs because no improvements over 10 subsequent epochs!
Epoch 52/100, Train Loss: 0.4580, Val Loss: 0.4065
The function took 674.02 seconds to execute.
No description has been provided for this image
Training Accuracy: 85.2775
Training Precision Scores: [18.17647059 98.58809802]
Training Recall Scores: [71.86046512 85.86382114]
Training F1 Scores: [29.01408451 91.78707224]

Validation Accuracy: 85.4361
Validation Precision Scores: [16.46778043 98.88320149]
Validation Recall Scores: [74.19354839 85.85858586]
Validation F1 Scores: [26.953125   91.91176471]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3166, Val Loss: 0.3152
Epoch 40/100, Train Loss: 0.2321, Val Loss: 0.2417
Early stopping triggered after 51 epochs because no improvements over 10 subsequent epochs!
Epoch 51/100, Train Loss: 0.1893, Val Loss: 0.2275
The function took 149.88 seconds to execute.
No description has been provided for this image
Training Accuracy: 92.1422
Training Precision Scores: [34.27856547 99.79054129]
Training Recall Scores: [95.58139535 91.99186992]
Training F1 Scores: [50.46040516 95.73264238]

Validation Accuracy: 91.9393
Validation Precision Scores: [29.64285714 99.56293706]
Validation Recall Scores: [89.24731183 92.04040404]
Validation F1 Scores: [44.50402145 95.65399958]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3062, Val Loss: 0.2841
Early stopping triggered after 26 epochs because no improvements over 10 subsequent epochs!
Epoch 26/100, Train Loss: 0.2738, Val Loss: 0.2806
The function took 78.63 seconds to execute.
No description has been provided for this image
Training Accuracy: 85.1120
Training Precision Scores: [20.9413009  99.59422365]
Training Recall Scores: [92.09302326 84.80691057]
Training F1 Scores: [34.12322275 91.60766233]

Validation Accuracy: 85.2804
Validation Precision Scores: [18.26280624 99.48088721]
Validation Recall Scores: [88.17204301 85.17171717]
Validation F1 Scores: [30.25830258 91.77187636]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3060, Val Loss: 0.2644
Epoch 40/100, Train Loss: 0.1650, Val Loss: 0.2089
Early stopping triggered after 43 epochs because no improvements over 10 subsequent epochs!
Epoch 43/100, Train Loss: 0.1599, Val Loss: 0.2396
The function took 171.90 seconds to execute.
No description has been provided for this image
Training Accuracy: 94.1967
Training Precision Scores: [41.51329243 99.7417133 ]
Training Recall Scores: [94.41860465 94.18699187]
Training F1 Scores: [57.67045455 96.88480033]

Validation Accuracy: 94.0031
Validation Precision Scores: [36.44444444 99.53051643]
Validation Recall Scores: [88.17204301 94.22222222]
Validation F1 Scores: [51.57232704 96.80365297]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3140, Val Loss: 0.2787
Epoch 40/100, Train Loss: 0.1474, Val Loss: 0.2000
Early stopping triggered after 52 epochs because no improvements over 10 subsequent epochs!
Epoch 52/100, Train Loss: 0.0984, Val Loss: 0.2022
The function took 358.10 seconds to execute.
No description has been provided for this image
Training Accuracy: 93.9241
Training Precision Scores: [40.65510597 99.91334489]
Training Recall Scores: [98.13953488 93.7398374 ]
Training F1 Scores: [57.49318801 96.72818792]

Validation Accuracy: 93.1075
Validation Precision Scores: [33.72093023 99.74025974]
Validation Recall Scores: [93.5483871  93.09090909]
Validation F1 Scores: [49.57264957 96.30094044]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4525, Val Loss: 0.4901
Early stopping triggered after 25 epochs because no improvements over 10 subsequent epochs!
Epoch 25/100, Train Loss: 0.4379, Val Loss: 0.3920
The function took 201.75 seconds to execute.
No description has been provided for this image
Training Accuracy: 93.0380
Training Precision Scores: [33.17591499 98.4187626 ]
Training Recall Scores: [65.34883721 94.24796748]
Training F1 Scores: [44.00939702 96.28822094]

Validation Accuracy: 93.1075
Validation Precision Scores: [29.61165049 98.64521592]
Validation Recall Scores: [65.59139785 94.14141414]
Validation F1 Scores: [40.80267559 96.34070705]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3163, Val Loss: 0.2705
Epoch 40/100, Train Loss: 0.1231, Val Loss: 0.2299
Early stopping triggered after 46 epochs because no improvements over 10 subsequent epochs!
Epoch 46/100, Train Loss: 0.0880, Val Loss: 0.2738
The function took 428.64 seconds to execute.
No description has been provided for this image
Training Accuracy: 95.9591
Training Precision Scores: [50.89820359 99.94700583]
Training Recall Scores: [98.8372093  95.83333333]
Training F1 Scores: [67.19367589 97.84695201]

Validation Accuracy: 95.2103
Validation Precision Scores: [42.42424242 99.62025316]
Validation Recall Scores: [90.32258065 95.39393939]
Validation F1 Scores: [57.73195876 97.46130031]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4553, Val Loss: 0.4246
Epoch 40/100, Train Loss: 0.3181, Val Loss: 0.3209
Epoch 60/100, Train Loss: 0.2522, Val Loss: 0.2788
Early stopping triggered after 78 epochs because no improvements over 10 subsequent epochs!
Epoch 78/100, Train Loss: 0.1235, Val Loss: 0.2309
The function took 1010.80 seconds to execute.
No description has been provided for this image
Training Accuracy: 95.6280
Training Precision Scores: [48.89663182 99.9043469 ]
Training Recall Scores: [97.90697674 95.52845528]
Training F1 Scores: [65.2207591  97.66741129]

Validation Accuracy: 94.5093
Validation Precision Scores: [38.46153846 99.44915254]
Validation Recall Scores: [86.02150538 94.82828283]
Validation F1 Scores: [53.15614618 97.08376422]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.2005, Val Loss: 0.2378
Epoch 40/100, Train Loss: 0.1220, Val Loss: 0.2017
Early stopping triggered after 52 epochs because no improvements over 10 subsequent epochs!
Epoch 52/100, Train Loss: 0.0898, Val Loss: 0.2328
The function took 157.13 seconds to execute.
No description has been provided for this image
Training Accuracy: 95.5794
Training Precision Scores: [48.63013699 99.95741963]
Training Recall Scores: [99.06976744 95.42682927]
Training F1 Scores: [65.237366   97.63959655]

Validation Accuracy: 94.4704
Validation Precision Scores: [38.60465116 99.57501062]
Validation Recall Scores: [89.24731183 94.66666667]
Validation F1 Scores: [53.8961039  97.05882353]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 18 epochs because no improvements over 10 subsequent epochs!
Epoch 18/100, Train Loss: 0.2937, Val Loss: 0.3128
The function took 73.48 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.8647
Training Precision Scores: [22.56716418 99.39499709]
Training Recall Scores: [87.90697674 86.81910569]
Training F1 Scores: [35.91448931 92.68239761]

Validation Accuracy: 86.9159
Validation Precision Scores: [20.4379562  99.58275382]
Validation Recall Scores: [90.32258065 86.78787879]
Validation F1 Scores: [33.33333333 92.74611399]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.1932, Val Loss: 0.2361
Early stopping triggered after 32 epochs because no improvements over 10 subsequent epochs!
Epoch 32/100, Train Loss: 0.0912, Val Loss: 0.2201
The function took 223.26 seconds to execute.
No description has been provided for this image
Training Accuracy: 94.4693
Training Precision Scores: [42.87190083 99.83874436]
Training Recall Scores: [96.51162791 94.3800813 ]
Training F1 Scores: [59.37052933 97.03270296]

Validation Accuracy: 93.9642
Validation Precision Scores: [35.77981651 99.36170213]
Validation Recall Scores: [83.87096774 94.34343434]
Validation F1 Scores: [50.1607717  96.78756477]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.2262, Val Loss: 0.2047
Early stopping triggered after 30 epochs because no improvements over 10 subsequent epochs!
Epoch 30/100, Train Loss: 0.1225, Val Loss: 0.2249
The function took 242.58 seconds to execute.
No description has been provided for this image
Training Accuracy: 92.9114
Training Precision Scores: [36.69642857 99.79234973]
Training Recall Scores: [95.58139535 92.79471545]
Training F1 Scores: [53.03225806 96.16640337]

Validation Accuracy: 92.2897
Validation Precision Scores: [30.48327138 99.5215311 ]
Validation Recall Scores: [88.17204301 92.44444444]
Validation F1 Scores: [45.3038674  95.85253456]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.1668, Val Loss: 0.2438
Early stopping triggered after 34 epochs because no improvements over 10 subsequent epochs!
Epoch 34/100, Train Loss: 0.1096, Val Loss: 0.2268
The function took 318.18 seconds to execute.
No description has been provided for this image
Training Accuracy: 94.3525
Training Precision Scores: [42.42424242 99.89224138]
Training Recall Scores: [97.6744186  94.20731707]
Training F1 Scores: [59.15492958 96.9665272 ]

Validation Accuracy: 94.0031
Validation Precision Scores: [36.56387665 99.57283212]
Validation Recall Scores: [89.24731183 94.18181818]
Validation F1 Scores: [51.875      96.80232558]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.3856, Val Loss: 0.3260
Early stopping triggered after 40 epochs because no improvements over 10 subsequent epochs!
Epoch 40/100, Train Loss: 0.1792, Val Loss: 0.3007
The function took 519.66 seconds to execute.
No description has been provided for this image
Training Accuracy: 93.1159
Training Precision Scores: [37.21144968 99.70610645]
Training Recall Scores: [93.72093023 93.08943089]
Training F1 Scores: [53.27164574 96.28422768]

Validation Accuracy: 93.0685
Validation Precision Scores: [32.21757322 99.31300988]
Validation Recall Scores: [82.79569892 93.45454545]
Validation F1 Scores: [46.38554217 96.29475437]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.1324, Val Loss: 0.2169
Early stopping triggered after 21 epochs because no improvements over 10 subsequent epochs!
Epoch 21/100, Train Loss: 0.1151, Val Loss: 0.2434
The function took 87.83 seconds to execute.
No description has been provided for this image
Training Accuracy: 91.1198
Training Precision Scores: [31.57492355 99.8103102 ]
Training Recall Scores: [96.04651163 90.90447154]
Training F1 Scores: [47.52589183 95.14945219]

Validation Accuracy: 91.5109
Validation Precision Scores: [28.37370242 99.51733216]
Validation Recall Scores: [88.17204301 91.63636364]
Validation F1 Scores: [42.93193717 95.41438788]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.1133, Val Loss: 0.2212
Early stopping triggered after 33 epochs because no improvements over 10 subsequent epochs!
Epoch 33/100, Train Loss: 0.0608, Val Loss: 0.3591
The function took 229.79 seconds to execute.
No description has been provided for this image
Training Accuracy: 95.9007
Training Precision Scores: [50.53380783 99.95756869]
Training Recall Scores: [99.06976744 95.76219512]
Training F1 Scores: [66.92851532 97.8149167 ]

Validation Accuracy: 94.8209
Validation Precision Scores: [40.0990099  99.49281488]
Validation Recall Scores: [87.09677419 95.11111111]
Validation F1 Scores: [54.91525424 97.25263375]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.1280, Val Loss: 0.3198
Early stopping triggered after 25 epochs because no improvements over 10 subsequent epochs!
Epoch 25/100, Train Loss: 0.0847, Val Loss: 0.3222
The function took 203.29 seconds to execute.
No description has been provided for this image
Training Accuracy: 94.3817
Training Precision Scores: [42.56825076 99.90302769]
Training Recall Scores: [97.90697674 94.22764228]
Training F1 Scores: [59.33756166 96.9823754 ]

Validation Accuracy: 93.6137
Validation Precision Scores: [34.63203463 99.44373128]
Validation Recall Scores: [86.02150538 93.8989899 ]
Validation F1 Scores: [49.38271605 96.5918537 ]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.1103, Val Loss: 0.3099
Early stopping triggered after 32 epochs because no improvements over 10 subsequent epochs!
Epoch 32/100, Train Loss: 0.0568, Val Loss: 0.4732
The function took 299.46 seconds to execute.
No description has been provided for this image
Training Accuracy: 96.1636
Training Precision Scores: [52.200489   99.96826069]
Training Recall Scores: [99.30232558 96.02642276]
Training F1 Scores: [68.42948718 97.95770267]

Validation Accuracy: 95.0156
Validation Precision Scores: [41.02564103 99.45217025]
Validation Recall Scores: [86.02150538 95.35353535]
Validation F1 Scores: [55.55555556 97.35973597]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.1158, Val Loss: 0.2760
Early stopping triggered after 27 epochs because no improvements over 10 subsequent epochs!
Epoch 27/100, Train Loss: 0.0781, Val Loss: 0.2489
The function took 351.50 seconds to execute.
No description has been provided for this image
Training Accuracy: 95.3651
Training Precision Scores: [47.40406321 99.89343564]
Training Recall Scores: [97.6744186  95.26422764]
Training F1 Scores: [63.82978723 97.52392842]

Validation Accuracy: 94.5093
Validation Precision Scores: [38.67924528 99.53310696]
Validation Recall Scores: [88.17204301 94.74747475]
Validation F1 Scores: [53.7704918  97.08134962]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 20 epochs because no improvements over 10 subsequent epochs!
Epoch 20/100, Train Loss: 0.0747, Val Loss: 0.3121
The function took 139.32 seconds to execute.
No description has been provided for this image
Training Accuracy: 93.7488
Training Precision Scores: [39.98109641 99.92401216]
Training Recall Scores: [98.37209302 93.54674797]
Training F1 Scores: [56.85483871 96.63027504]

Validation Accuracy: 92.8349
Validation Precision Scores: [31.72690763 99.3962915 ]
Validation Recall Scores: [84.94623656 93.13131313]
Validation F1 Scores: [46.19883041 96.161869  ]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.0726, Val Loss: 0.2646
Early stopping triggered after 21 epochs because no improvements over 10 subsequent epochs!
Epoch 21/100, Train Loss: 0.0700, Val Loss: 0.2781
The function took 170.19 seconds to execute.
No description has been provided for this image
Training Accuracy: 94.6446
Training Precision Scores: [43.68421053 99.83905579]
Training Recall Scores: [96.51162791 94.56300813]
Training F1 Scores: [60.14492754 97.12943633]

Validation Accuracy: 93.7305
Validation Precision Scores: [35.59322034 99.61406518]
Validation Recall Scores: [90.32258065 93.85858586]
Validation F1 Scores: [51.06382979 96.6507177 ]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.0750, Val Loss: 0.2874
Early stopping triggered after 23 epochs because no improvements over 10 subsequent epochs!
Epoch 23/100, Train Loss: 0.0578, Val Loss: 0.3331
The function took 217.04 seconds to execute.
No description has been provided for this image
Training Accuracy: 93.7001
Training Precision Scores: [39.81220657 99.93481803]
Training Recall Scores: [98.60465116 93.48577236]
Training F1 Scores: [56.72240803 96.60278288]

Validation Accuracy: 93.3022
Validation Precision Scores: [34.26294821 99.6978852 ]
Validation Recall Scores: [92.47311828 93.33333333]
Validation F1 Scores: [50.         96.41068447]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.1297, Val Loss: 0.3066
Early stopping triggered after 32 epochs because no improvements over 10 subsequent epochs!
Epoch 32/100, Train Loss: 0.0745, Val Loss: 0.3719
The function took 421.73 seconds to execute.
No description has been provided for this image
Training Accuracy: 96.8744
Training Precision Scores: [57.47599451 99.8847081 ]
Training Recall Scores: [97.44186047 96.8495935 ]
Training F1 Scores: [72.30371009 98.34373871]

Validation Accuracy: 96.1059
Validation Precision Scores: [47.85276074 99.37629938]
Validation Recall Scores: [83.87096774 96.56565657]
Validation F1 Scores: [60.9375     97.95081967]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 18 epochs because no improvements over 10 subsequent epochs!
Epoch 18/100, Train Loss: 0.0536, Val Loss: 0.3013
The function took 150.94 seconds to execute.
No description has been provided for this image
Training Accuracy: 92.1227
Training Precision Scores: [34.42892358 99.87849332]
Training Recall Scores: [97.44186047 91.8902439 ]
Training F1 Scores: [50.88038859 95.71799079]

Validation Accuracy: 91.6667
Validation Precision Scores: [29.06574394 99.60508995]
Validation Recall Scores: [90.32258065 91.71717172]
Validation F1 Scores: [43.97905759 95.49852756]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Early stopping triggered after 20 epochs because no improvements over 10 subsequent epochs!
Epoch 20/100, Train Loss: 0.0398, Val Loss: 0.4037
The function took 195.21 seconds to execute.
No description has been provided for this image
Training Accuracy: 95.9494
Training Precision Scores: [50.84541063 99.90468121]
Training Recall Scores: [97.90697674 95.86382114]
Training F1 Scores: [66.93163752 97.84254745]

Validation Accuracy: 94.9377
Validation Precision Scores: [40.7960199  99.53527672]
Validation Recall Scores: [88.17204301 95.19191919]
Validation F1 Scores: [55.78231293 97.31515903]

-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 512
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.0408, Val Loss: 0.3947
Early stopping triggered after 25 epochs because no improvements over 10 subsequent epochs!
Epoch 25/100, Train Loss: 0.0339, Val Loss: 0.4991
The function took 339.62 seconds to execute.
No description has been provided for this image
Training Accuracy: 97.8384
Training Precision Scores: [66.19937695 99.94806813]
Training Recall Scores: [98.8372093  97.79471545]
Training F1 Scores: [79.29104478 98.85966715]

Validation Accuracy: 96.5343
Validation Precision Scores: [51.2987013  99.42004971]
Validation Recall Scores: [84.94623656 96.96969697]
Validation F1 Scores: [63.96761134 98.17958683]

In [18]:
#16 and hidden dim 512

train_dataset = TimeSeriesDataset(X_train, y_train)
val_dataset = TimeSeriesDataset(X_val, y_val)
test_dataset = TimeSeriesDataset(X_test, y_test)

train_loader = DataLoader(train_dataset, batch_size=64, collate_fn=collate_fn,shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)
test_loader = DataLoader(test_dataset, batch_size=64, collate_fn=collate_fn, shuffle=False)

embedding_size=16
hidden_dim=512

vocabulary_size = len(vectorizer.vocabulary_) + 2  # Add 2 for out-of-vocabulary and padding
output_size = 2

model = LSTMClassifier(vocabulary_size, embedding_size, hidden_dim, output_size)
model = model.to(device)

class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)  # Computing class weights
class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)

optimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.01)
num_epochs = 100
training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, criterion, num_epochs)

test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model(model,test_loader,device)
stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')
Training:   0%|          | 0/100 [00:00<?, ?it/s]
Epoch 20/100, Train Loss: 0.4789, Val Loss: 0.3905
Epoch 40/100, Train Loss: 0.4123, Val Loss: 0.3309
Epoch 60/100, Train Loss: 0.1518, Val Loss: 0.2850
Early stopping triggered after 61 epochs because no improvements over 10 subsequent epochs!
Epoch 61/100, Train Loss: 0.1539, Val Loss: 0.2404
The function took 787.23 seconds to execute.
No description has been provided for this image
Test Accuracy: 91.9445
Test Precision Scores: [27.734375   99.43777541]
Test Recall Scores: [85.2        92.18199746]
Test F1 Scores: [41.84675835 95.67251462]

Out[18]:
<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7a939e3c8690>
No description has been provided for this image
No description has been provided for this image

Task 4: GNN¶

In [4]:
from datetime import time
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import accuracy_score, precision_score, f1_score, recall_score, ConfusionMatrixDisplay
import torch
from sklearn.model_selection import train_test_split
from torch import nn, optim
from tqdm.notebook import tqdm
import time
!pip install torch_geometric
from torch_geometric.data import Data, Dataset
from sklearn.preprocessing import OneHotEncoder
import matplotlib.pyplot as plt
from torch_geometric.nn import GCNConv, SAGEConv, GATConv, global_mean_pool
import torch.nn.functional as F
import torch_geometric
from sklearn.utils.class_weight import compute_class_weight
import networkx as nx
import random

# Set seeds
seed = 69
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)

# Set GPU
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"The device is set to: {device}")
Collecting torch_geometric
  Downloading torch_geometric-2.6.1-py3-none-any.whl.metadata (63 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 63.1/63.1 kB 2.3 MB/s eta 0:00:00
Requirement already satisfied: aiohttp in /usr/local/lib/python3.11/dist-packages (from torch_geometric) (3.12.13)
Requirement already satisfied: fsspec in /usr/local/lib/python3.11/dist-packages (from torch_geometric) (2025.5.1)
Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/dist-packages (from torch_geometric) (3.1.6)
Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (from torch_geometric) (1.26.4)
Requirement already satisfied: psutil>=5.8.0 in /usr/local/lib/python3.11/dist-packages (from torch_geometric) (7.0.0)
Requirement already satisfied: pyparsing in /usr/local/lib/python3.11/dist-packages (from torch_geometric) (3.0.9)
Requirement already satisfied: requests in /usr/local/lib/python3.11/dist-packages (from torch_geometric) (2.32.4)
Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (from torch_geometric) (4.67.1)
Requirement already satisfied: aiohappyeyeballs>=2.5.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->torch_geometric) (2.6.1)
Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/dist-packages (from aiohttp->torch_geometric) (1.3.2)
Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->torch_geometric) (25.3.0)
Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/dist-packages (from aiohttp->torch_geometric) (1.7.0)
Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/dist-packages (from aiohttp->torch_geometric) (6.6.3)
Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->torch_geometric) (0.3.2)
Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->torch_geometric) (1.20.1)
Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/dist-packages (from jinja2->torch_geometric) (3.0.2)
Requirement already satisfied: mkl_fft in /usr/local/lib/python3.11/dist-packages (from numpy->torch_geometric) (1.3.8)
Requirement already satisfied: mkl_random in /usr/local/lib/python3.11/dist-packages (from numpy->torch_geometric) (1.2.4)
Requirement already satisfied: mkl_umath in /usr/local/lib/python3.11/dist-packages (from numpy->torch_geometric) (0.1.1)
Requirement already satisfied: mkl in /usr/local/lib/python3.11/dist-packages (from numpy->torch_geometric) (2025.2.0)
Requirement already satisfied: tbb4py in /usr/local/lib/python3.11/dist-packages (from numpy->torch_geometric) (2022.2.0)
Requirement already satisfied: mkl-service in /usr/local/lib/python3.11/dist-packages (from numpy->torch_geometric) (2.4.1)
Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests->torch_geometric) (3.4.2)
Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests->torch_geometric) (3.10)
Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests->torch_geometric) (2.5.0)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests->torch_geometric) (2025.6.15)
Requirement already satisfied: intel-openmp<2026,>=2024 in /usr/local/lib/python3.11/dist-packages (from mkl->numpy->torch_geometric) (2024.2.0)
Requirement already satisfied: tbb==2022.* in /usr/local/lib/python3.11/dist-packages (from mkl->numpy->torch_geometric) (2022.2.0)
Requirement already satisfied: tcmlib==1.* in /usr/local/lib/python3.11/dist-packages (from tbb==2022.*->mkl->numpy->torch_geometric) (1.4.0)
Requirement already satisfied: intel-cmplr-lib-rt in /usr/local/lib/python3.11/dist-packages (from mkl_umath->numpy->torch_geometric) (2024.2.0)
Requirement already satisfied: intel-cmplr-lib-ur==2024.2.0 in /usr/local/lib/python3.11/dist-packages (from intel-openmp<2026,>=2024->mkl->numpy->torch_geometric) (2024.2.0)
Downloading torch_geometric-2.6.1-py3-none-any.whl (1.1 MB)
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.1/1.1 MB 29.4 MB/s eta 0:00:0000:01
Installing collected packages: torch_geometric
Successfully installed torch_geometric-2.6.1
The device is set to: cuda
In [5]:
def seq_to_edge_lists(seq):
    """
    Dato seq = [c0, c1, …, cN-1], costruisce:
      sources = [m(c0), m(c1), …, m(cN-2)]
      dests   = [m(c1), m(c2), …, m(cN-1)]
    dove m(x) è l'indice interno assegnato in ordine di prima apparizione.

    Esempio:
      seq = [3,7,1,3,1,3,2]
      mapping = {3:0, 7:1, 1:2, 2:3}
      sources  = [0,1,2,0,2,0]
      dests    = [1,2,0,2,0,3]
    """
    mapping = {}     # comando_originale -> nuovo_indice
    next_id = 0

    # 1) Crea la mappatura comando->indice
    for cmd in seq:
        if cmd not in mapping:
            mapping[cmd] = next_id
            next_id += 1

    # 2) Costruisci le liste di archi (consecutive pairs)
    sources = []
    dests   = []
    for u, v in zip(seq, seq[1:]):  # zip(seq, seq[1:]) è equivalente a fare for i in range(len(seq) - 1) e prendere u = seq[i] e v = seq[i + 1]
        sources.append(mapping[u])
        dests.append(mapping[v])

    return sources, dests

class GraphDataset(Dataset):
    def __init__(self, data, labels, embedds):
        """
        Initialize dataset by pre-processing all graphs into PyG `Data` format.

        Args:
            data (list of np.array): List of sequences.
            labels (list of int): List of graph labels.
            embeds embedding created outside.
        """
        super().__init__()
        self.graphs = []  # Store processed graphs

        # Create embedding layer with the given vocabulary size and embedding size
        self.embedding_layer = embedds

        for i in range(len(data)):
            # Convert adjacency matrix to edge index
            edge_index = seq_to_edge_lists(data[i])  # shape = (2, E)
            edge_index = torch.tensor(edge_index, dtype=torch.long)

            # Convert node features to embeddings
            x = torch.tensor(data[i], dtype=torch.long)  # Node feature indices
            x = self.embedding_layer(x)  # Convert indices to embeddings
            # Ensure `y` is a single integer label
            #y = torch.tensor(labels[i].nonzero()[0], dtype=torch.long).unsqueeze(0)  # Class index of the graph
            # labels[i].nonzero()[0] to encode the label from a one-hot-encoding representation into a numerical version
            y = labels[i].nonzero()[0].clone().detach().to(dtype=torch.long).unsqueeze(0)


            # Create Data object and store it
            self.graphs.append(Data(x=x, edge_index=edge_index, y=y))

    def len(self):
        return len(self.graphs)

    def get(self, idx):
        return self.graphs[idx]  # Return a pre-processed graph

Preprocessing¶

In [6]:
data_1 = pd.read_json("/kaggle/input/lab22222/train.json").drop(columns='md5hash')
data_2 = pd.read_json("/kaggle/input/lab22222/test.json").drop(columns='md5hash')

previous_length = len(data_1)
data_1 = data_1.drop_duplicates(subset=["api_call_sequence"])
print(f"Removed {previous_length - len(data_1)} duplicates \nDataset length: {len(data_1)}")

previous_length = len(data_2)
data_2 = data_2.drop_duplicates(subset=["api_call_sequence"])
print(f"Removed {previous_length - len(data_2)} duplicates \nDataset length: {len(data_2)}")

X = data_1.iloc[:, :-1].values # Remove label column
X = [x[0] for x in X]
y = data_1.iloc[:, -1].values
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=seed)  # 80% of the data is used for training
X_test = data_2.iloc[:, :-1].values # Remove label column
X_test = [x[0] for x in X_test]
y_test = data_2.iloc[:, -1].values

X_train_str = [str(x) for x in X_train]
vectorizer = CountVectorizer()

# Computing the matrix representing the words contained in each api sequence
document_term_matrix_training = vectorizer.fit_transform(X_train_str)  # The vocabulary is computed on the training set and is used to compute the matrix for the other datasets

# Assigning sequential IDs to training set
for i in range(0, len(X_train)):
    row = X_train[i]
    for j in range(len(row)):
        row[j] = vectorizer.vocabulary_[row[j].lower()]
    X_train[i] = row

# Assigning sequential IDs to validation set
for i in range(0, len(X_val)):
    row = X_val[i]
    for j in range(len(row)):
        if row[j].lower() not in vectorizer.vocabulary_.keys():  # To manage the out-of-vocabulary words
            row[j] = max(vectorizer.vocabulary_.values()) + 1
        else:
            row[j] = vectorizer.vocabulary_[row[j].lower()]
    X_val[i] = row

# Assigning sequential IDs to testing set
for i in range(0, len(X_test)):
    row = X_test[i]
    for j in range(len(row)):
        if row[j].lower() not in vectorizer.vocabulary_.keys():  # To manage the out-of-vocabulary words
            row[j] = max(vectorizer.vocabulary_.values()) + 1
        else:
            row[j] = vectorizer.vocabulary_[row[j].lower()]
    X_test[i] = row

def stats(accuracy, precision, recall, f1, description):
    print(f'{description} Accuracy: {accuracy:.4f}')
    print(f'{description} Precision Scores: {precision}')
    print(f'{description} Recall Scores: {recall}')
    print(f'{description} F1 Scores: {f1}\n')
Removed 17875 duplicates 
Dataset length: 12838
Removed 5814 duplicates 
Dataset length: 7349

Modifying the labels using one hot encoding¶

In [7]:
# Reshape the labels to ensure compatibility with OneHotEncoder
y_train_original = y_train
y_val_original = y_val
y_test_original = y_test
y_train_reshaped = np.array(y_train).reshape(-1, 1).astype('long')
y_val_reshaped = np.array(y_val).reshape(-1, 1).astype('long')
y_test_reshaped = np.array(y_test).reshape(-1, 1).astype('long')

# Initialize the OneHotEncoder
encoder = OneHotEncoder(sparse_output=False)

# Fit and transform the labels
y_train = torch.tensor(encoder.fit_transform(y_train_reshaped)).to(torch.long)
y_val = torch.tensor(encoder.transform(y_val_reshaped)).to(torch.long)
y_test = torch.tensor(encoder.transform(y_test_reshaped)).to(torch.long)
In [8]:
# Create embedding layer with the given vocabulary size and embedding size
embedding_size = 8 # initially 8 but then we modify it
vocabulary_size = len(vectorizer.vocabulary_) + 1  # Add 1 for out-of-vocabulary
embeddings = nn.Embedding(vocabulary_size, embedding_size)
X_train_graph = GraphDataset(X_train, y_train, embeddings)
X_val_graph = GraphDataset(X_val, y_val, embeddings)
X_test_graph = GraphDataset(X_test, y_test, embeddings)
In [9]:
def plot_graph_simple(data):
    edge_index = data.edge_index.numpy()

    # Create directed graph
    G = nx.DiGraph()

    # Calculate unique nodes strictly from edge_index
    unique_nodes = set(np.concatenate(edge_index))
    for node in unique_nodes:
        G.add_node(node)  # Add only nodes present in edges

    # Add edges to the graph
    for i in range(edge_index.shape[1]):
        source = edge_index[0, i]
        target = edge_index[1, i]
        G.add_edge(source, target)

    # Print graph info
    print(f"NetworkX graph has {G.number_of_nodes()} nodes and {G.number_of_edges()} edges")

    # Plot with different layout options
    plt.figure(figsize=(10, 8))
    pos = nx.kamada_kawai_layout(G)  # Use kamada_kawai_layout for better structure

    # Draw with high visibility settings
    nx.draw(G, pos,
            with_labels=True,
            node_color='skyblue',
            node_size=700,
            font_size=10,
            font_weight='bold',
            arrowsize=20,
            width=2,
            edge_color='red',
            arrows=True)

    plt.title(f"Graph Visualization - Label: {data.y.item()}")
    plt.tight_layout()
    plt.show()

# Re-run the plot
plot_graph_simple(X_train_graph[4])
NetworkX graph has 26 nodes and 54 edges
/tmp/ipykernel_36/2669503328.py:38: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect.
  plt.tight_layout()
No description has been provided for this image

GNN¶

In [10]:
def training_loop(model, train_loader, val_loader, train_dataset, val_dataset, device, optimizer, num_epochs, criterion, patience=10, min_delta=0.001):
    start_time = time.time()
    train_losses = []
    val_losses = []

    best_val_loss = float('inf')
    epochs_no_improvement = 0

    # Training loop
    for epoch in tqdm(range(num_epochs)):  # Train for epochs
        train_loss = 0
        val_loss = 0
        model.train()  # Set model to training mode
        for batch in train_loader:
            batch = batch.to(device)  # Move batch to GPU
            optimizer.zero_grad()  # Clear previous gradients
            outputs = model(batch)  # Forward pass
            loss = criterion(outputs, batch.y.squeeze())  # Compute loss
            loss.backward(retain_graph=True)  # Backpropagation
            optimizer.step()  # Update model parameters
            train_loss += loss.item() * batch.size(0)
        train_loss /= len(train_dataset)
        train_losses.append(train_loss)  # Store training loss

        # Compute Validation loss during training
        model.eval()  # Set model to evaluation mode
        with torch.no_grad():  # Disable gradient computation for validation
            for batch in val_loader:
                batch = batch.to(device)  # Move batch to GPU
                val_outputs = model(batch)  # Forward pass
                loss = criterion(val_outputs, batch.y.squeeze())  # Compute validation loss. .squeeze() is used to reduce from 2D to 1D vector
                val_loss += loss.item() * batch.size(0)
            val_loss /= len(val_dataset)
            val_losses.append(val_loss)  # Store validation loss

        # Early stopping
        if best_val_loss - val_loss > min_delta:
            best_val_loss = val_loss
            epochs_no_improvement = 0
            torch.save(model.state_dict(), 'best_model.pth')  # Saving the best model
        else:
            epochs_no_improvement += 1
            if epochs_no_improvement >= patience:
                print(f'Early stopping triggered after {epoch + 1} epochs because no improvements over {patience} subsequent epochs!')
                print(f'Epoch {epoch+1}/{num_epochs}, Train Loss: {train_losses[-1]:.4f}, Val Loss: {val_losses[-1]:.4f}')
                break  # Stop training

        if (epoch + 1) % 5 == 0:
            print(f'Epoch {epoch+1}/{num_epochs}, Train Loss: {train_losses[-1]:.4f}, Val Loss: {val_losses[-1]:.4f}')

    end_time = time.time()
    elapsed_time = end_time - start_time

    print(f'The function took {elapsed_time:.2f} seconds to execute.')

    # Restore best model weights
    model.load_state_dict(torch.load('best_model.pth', map_location=device))

    # Plot the training and validation loss
    plt.figure(figsize=(10, 5))
    plt.plot(train_losses, label='Train Loss')
    plt.plot(val_losses, label='Validation Loss')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.title('Training and Validation Loss')
    plt.legend()
    plt.grid()
    plt.show()

    return

def testing_model(model, dataloader, device):
    model.eval()  # Set the model to evaluation mode
    all_labels = []
    all_predictions = []

    with torch.no_grad():  # Disable gradient computation for efficiency
        for batch in dataloader:
            batch = batch.to(device)  # Move batch to GPU
            outputs = model(batch)
            _, predicted = torch.max(outputs, 1)  # Get the class with highest score
            all_labels.extend(batch.y.squeeze().cpu().numpy())
            all_predictions.extend(predicted.cpu().numpy())

    accuracy = accuracy_score(all_labels, all_predictions) * 100
    precision = precision_score(all_labels, all_predictions, average=None) * 100
    recall = recall_score(all_labels, all_predictions, average=None) * 100
    f1 = f1_score(all_labels, all_predictions, average=None) * 100

    return accuracy, precision, recall, f1, all_labels, all_predictions

def testing_model_with_threshold(model, dataloader, device, threshold=0.5):
    model.eval()
    all_labels = []
    all_predictions = []
    with torch.no_grad():
        for batch in dataloader:
            batch = batch.to(device)
            outputs = model(batch)
            probabilities = F.softmax(outputs, dim=1)  # Convert logits to probabilities
            predicted = (probabilities[:, 0] > threshold).long()  # 1 if p_class0 > threshold, else 0
            predicted_classes = 1 - predicted  # If predicted=1, class=0; if predicted=0, class=1
            all_labels.extend(batch.y.squeeze().cpu().numpy())
            all_predictions.extend(predicted_classes.cpu().numpy())
    accuracy = accuracy_score(all_labels, all_predictions) * 100
    precision = precision_score(all_labels, all_predictions, average=None) * 100
    recall = recall_score(all_labels, all_predictions, average=None) * 100
    f1 = f1_score(all_labels, all_predictions, average=None) * 100
    return accuracy, precision, recall, f1, all_labels, all_predictions

class GCN(torch.nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(GCN, self).__init__()
        self.conv1 = GCNConv(input_dim, hidden_dim)
        self.conv2 = GCNConv(hidden_dim, hidden_dim)
        self.fc = torch.nn.Linear(hidden_dim, output_dim)
        self.bn = torch.nn.BatchNorm1d(hidden_dim)
        self.dropout = torch.nn.Dropout(0.4)

    def forward(self, data):
        x, edge_index, batch = data.x, data.edge_index, data.batch
        x = F.relu(self.conv1(x, edge_index))
        x = self.dropout(x)
        x = self.bn(x)
        x = F.relu(self.conv2(x, edge_index))
        x = global_mean_pool(x, batch)  # Aggregate node features from Node to Graph features
        x = self.fc(x)
        return x

class GraphSAGE(torch.nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(GraphSAGE, self).__init__()
        self.conv1 = SAGEConv(input_dim, hidden_dim)
        self.conv2 = SAGEConv(hidden_dim, hidden_dim)
        self.fc = torch.nn.Linear(hidden_dim, output_dim)
        self.dropout = torch.nn.Dropout(0.4)

    def forward(self, data):
        x, edge_index, batch = data.x, data.edge_index, data.batch
        x = F.relu(self.conv1(x, edge_index))
        x = self.dropout(x)
        x = F.relu(self.conv2(x, edge_index))
        x = global_mean_pool(x, batch) # Aggregate node features. From Node to Graph Features
        x = self.fc(x)
        return x

class GAT(torch.nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim, heads=4):
        super(GAT, self).__init__()
        self.conv1 = GATConv(input_dim, hidden_dim, heads=heads, concat=True)
        self.conv2 = GATConv(hidden_dim * heads, hidden_dim, heads=1, concat=False)
        self.fc = torch.nn.Linear(hidden_dim, output_dim)

    def forward(self, data):
        x, edge_index, batch = data.x, data.edge_index, data.batch
        x = F.elu(self.conv1(x, edge_index))
        x = F.elu(self.conv2(x, edge_index))
        x = global_mean_pool(x, batch) # Aggregate node features. From Node to Graph Features
        x = self.fc(x)
        return x

Simple GCN¶

In [12]:
embedding_sizes = [8,16,32,64,128,256]
hidden_dims = [8,16,32,64,128,256,320, 384, 512]

for i in range(len(embedding_sizes)):
    for j in range(i,len(hidden_dims)):
        embedding_size=embedding_sizes[i]
        hidden_dim=hidden_dims[j]
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print(f"Trying with embedding size {embedding_size} and hidden dim {hidden_dim}")

        # Create embedding layer with the given vocabulary size and embedding size
        vocabulary_size = len(vectorizer.vocabulary_) + 1  # Add 1 for out-of-vocabulary
        embeddings = nn.Embedding(vocabulary_size, embedding_size)
        X_train_graph = GraphDataset(X_train, y_train, embeddings)
        X_val_graph = GraphDataset(X_val, y_val, embeddings)
        X_test_graph = GraphDataset(X_test, y_test, embeddings)
        
        # Create DataLoader for batching
        # NOTICE: This data loader is from torch_geometric.loader!! It is not from torch.utils.data
        batch_size = 64
        train_loader = torch_geometric.loader.DataLoader(X_train_graph, batch_size=batch_size, shuffle=True)
        val_loader = torch_geometric.loader.DataLoader(X_val_graph, batch_size=batch_size, shuffle=False)
        test_loader = torch_geometric.loader.DataLoader(X_test_graph, batch_size=batch_size, shuffle=False)
        
        class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train_original), y=y_train_original)  # Computing class weights
        class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
        criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
        num_epochs = 50
        model = GCN(input_dim=embedding_size, hidden_dim=hidden_dim, output_dim=2).to(device)
        optimizer = optim.AdamW(model.parameters(), lr=0.01, weight_decay=0.01)
        training_loop(model, train_loader, val_loader, X_train_graph, X_val_graph, device, optimizer, num_epochs, criterion)
        
        train_accuracy, train_precisionscores, train_recallscores, train_f1scores, _, _ = testing_model(model,train_loader,device)
        val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model(model,val_loader,device)
        #test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model(model,test_loader,device)
        
        stats(train_accuracy, train_precisionscores, train_recallscores, train_f1scores, 'Training')
        stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
        #stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')
        #ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
        #ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')
        
        # Test with different thresholds
        for threshold in [0.5, 0.6, 0.7, 0.8, 0.9]:
            print(f"Threshold: {threshold}")
            val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model_with_threshold(model,val_loader,device,threshold=threshold)
            stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
            print()
-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 8
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 46.1322, Val Loss: 44.1200
Epoch 10/50, Train Loss: 43.3972, Val Loss: 43.6996
Epoch 15/50, Train Loss: 41.7698, Val Loss: 43.1021
Epoch 20/50, Train Loss: 43.0420, Val Loss: 42.7623
Early stopping triggered after 22 epochs because no improvements over 10 subsequent epochs!
Epoch 22/50, Train Loss: 43.3233, Val Loss: 41.6739
The function took 30.87 seconds to execute.
No description has been provided for this image
Training Accuracy: 72.8432
Training Precision Scores: [11.05975569 98.68802652]
Training Recall Scores: [77.90697674 72.62195122]
Training F1 Scores: [19.36976005 83.67191616]

Validation Accuracy: 70.8333
Validation Precision Scores: [ 9.2039801  98.92290249]
Validation Recall Scores: [79.56989247 70.50505051]
Validation F1 Scores: [16.49944259 82.33073838]

Threshold: 0.5
Validation Accuracy: 70.8333
Validation Precision Scores: [ 9.2039801  98.92290249]
Validation Recall Scores: [79.56989247 70.50505051]
Validation F1 Scores: [16.49944259 82.33073838]


Threshold: 0.6
Validation Accuracy: 79.5561
Validation Precision Scores: [11.5658363  98.60418744]
Validation Recall Scores: [69.89247312 79.91919192]
Validation F1 Scores: [19.84732824 88.28386521]


Threshold: 0.7
Validation Accuracy: 86.9159
Validation Precision Scores: [13.06990881 97.76686021]
Validation Recall Scores: [46.23655914 88.44444444]
Validation F1 Scores: [20.37914692 92.87229529]


Threshold: 0.8
Validation Accuracy: 92.5623
Validation Precision Scores: [15.         97.03459638]
Validation Recall Scores: [22.58064516 95.19191919]
Validation F1 Scores: [18.02575107 96.10442586]


Threshold: 0.9
Validation Accuracy: 95.5607
Validation Precision Scores: [20.         96.60481642]
Validation Recall Scores: [ 7.52688172 98.86868687]
Validation F1 Scores: [10.9375     97.72364217]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 16
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 42.9639, Val Loss: 42.7165
Epoch 10/50, Train Loss: 39.4247, Val Loss: 38.9916
Epoch 15/50, Train Loss: 38.9545, Val Loss: 42.3849
Epoch 20/50, Train Loss: 38.6360, Val Loss: 37.6971
Epoch 25/50, Train Loss: 37.5386, Val Loss: 36.4397
Epoch 30/50, Train Loss: 37.9233, Val Loss: 37.0997
Epoch 35/50, Train Loss: 37.3246, Val Loss: 35.8904
Epoch 40/50, Train Loss: 36.9472, Val Loss: 36.9899
Early stopping triggered after 43 epochs because no improvements over 10 subsequent epochs!
Epoch 43/50, Train Loss: 36.8017, Val Loss: 38.0919
The function took 60.72 seconds to execute.
No description has been provided for this image
Training Accuracy: 82.4343
Training Precision Scores: [15.92261905 98.67942816]
Training Recall Scores: [74.65116279 82.77439024]
Training F1 Scores: [26.24693377 90.02984415]

Validation Accuracy: 82.2040
Validation Precision Scores: [14.453125   99.07587549]
Validation Recall Scores: [79.56989247 82.3030303 ]
Validation F1 Scores: [24.46280992 89.91392629]

Threshold: 0.5
Validation Accuracy: 82.2040
Validation Precision Scores: [14.453125   99.07587549]
Validation Recall Scores: [79.56989247 82.3030303 ]
Validation F1 Scores: [24.46280992 89.91392629]


Threshold: 0.6
Validation Accuracy: 88.0062
Validation Precision Scores: [15.87301587 98.09143364]
Validation Recall Scores: [53.76344086 89.29292929]
Validation F1 Scores: [24.50980392 93.4856176 ]


Threshold: 0.7
Validation Accuracy: 91.9003
Validation Precision Scores: [17.14285714 97.36732135]
Validation Recall Scores: [32.25806452 94.14141414]
Validation F1 Scores: [22.3880597  95.72719803]


Threshold: 0.8
Validation Accuracy: 95.0545
Validation Precision Scores: [20.68965517 96.77290837]
Validation Recall Scores: [12.90322581 98.14141414]
Validation F1 Scores: [15.89403974 97.45235707]


Threshold: 0.9
Validation Accuracy: 95.7555
Validation Precision Scores: [23.33333333 96.61150512]
Validation Recall Scores: [ 7.52688172 99.07070707]
Validation F1 Scores: [11.38211382 97.8256533 ]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 32
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 40.6945, Val Loss: 37.1508
Epoch 10/50, Train Loss: 37.8760, Val Loss: 34.2618
Epoch 15/50, Train Loss: 36.9977, Val Loss: 32.8120
Epoch 20/50, Train Loss: 35.8666, Val Loss: 37.7323
Early stopping triggered after 25 epochs because no improvements over 10 subsequent epochs!
Epoch 25/50, Train Loss: 36.9255, Val Loss: 34.5277
The function took 34.81 seconds to execute.
No description has been provided for this image
Training Accuracy: 82.9503
Training Precision Scores: [16.11082606 98.60593679]
Training Recall Scores: [73.02325581 83.38414634]
Training F1 Scores: [26.39764607 90.35846044]

Validation Accuracy: 83.5670
Validation Precision Scores: [15.07430998 98.95088221]
Validation Recall Scores: [76.34408602 83.83838384]
Validation F1 Scores: [25.17730496 90.76990376]

Threshold: 0.5
Validation Accuracy: 83.5670
Validation Precision Scores: [15.07430998 98.95088221]
Validation Recall Scores: [76.34408602 83.83838384]
Validation F1 Scores: [25.17730496 90.76990376]


Threshold: 0.6
Validation Accuracy: 89.4860
Validation Precision Scores: [19.58762887 98.41897233]
Validation Recall Scores: [61.29032258 90.54545455]
Validation F1 Scores: [29.6875     94.31818182]


Threshold: 0.7
Validation Accuracy: 92.9128
Validation Precision Scores: [23.35329341 97.75093711]
Validation Recall Scores: [41.93548387 94.82828283]
Validation F1 Scores: [30.         96.26743232]


Threshold: 0.8
Validation Accuracy: 95.0156
Validation Precision Scores: [26.02739726 97.03406814]
Validation Recall Scores: [20.43010753 97.81818182]
Validation F1 Scores: [22.89156627 97.42454728]


Threshold: 0.9
Validation Accuracy: 96.1838
Validation Precision Scores: [40.         96.73613842]
Validation Recall Scores: [10.75268817 99.39393939]
Validation F1 Scores: [16.94915254 98.04703069]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 64
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 42.0951, Val Loss: 40.8983
Epoch 10/50, Train Loss: 40.5234, Val Loss: 38.8316
Epoch 15/50, Train Loss: 37.3142, Val Loss: 35.2674
Epoch 20/50, Train Loss: 35.8519, Val Loss: 35.5465
Epoch 25/50, Train Loss: 33.6738, Val Loss: 34.7888
Early stopping triggered after 27 epochs because no improvements over 10 subsequent epochs!
Epoch 27/50, Train Loss: 34.1043, Val Loss: 33.0375
The function took 37.50 seconds to execute.
No description has been provided for this image
Training Accuracy: 82.1422
Training Precision Scores: [15.78947368 98.71014845]
Training Recall Scores: [75.34883721 82.43902439]
Training F1 Scores: [26.10797744 89.84383653]

Validation Accuracy: 81.8536
Validation Precision Scores: [13.78640777 98.92839747]
Validation Recall Scores: [76.34408602 82.06060606]
Validation F1 Scores: [23.35526316 89.70848057]

Threshold: 0.5
Validation Accuracy: 81.8536
Validation Precision Scores: [13.78640777 98.92839747]
Validation Recall Scores: [76.34408602 82.06060606]
Validation F1 Scores: [23.35526316 89.70848057]


Threshold: 0.6
Validation Accuracy: 92.1340
Validation Precision Scores: [25.33936652 98.42351939]
Validation Recall Scores: [60.21505376 93.33333333]
Validation F1 Scores: [35.66878981 95.81086686]


Threshold: 0.7
Validation Accuracy: 95.2103
Validation Precision Scores: [36.36363636 97.84377543]
Validation Recall Scores: [43.01075269 97.17171717]
Validation F1 Scores: [39.408867   97.50658828]


Threshold: 0.8
Validation Accuracy: 96.1838
Validation Precision Scores: [45.28301887 97.25646123]
Validation Recall Scores: [25.80645161 98.82828283]
Validation F1 Scores: [32.87671233 98.03607214]


Threshold: 0.9
Validation Accuracy: 96.5343
Validation Precision Scores: [66.66666667 96.67449139]
Validation Recall Scores: [ 8.60215054 99.83838384]
Validation F1 Scores: [15.23809524 98.230968  ]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 128
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 37.9157, Val Loss: 38.1506
Epoch 10/50, Train Loss: 34.5610, Val Loss: 35.3705
Epoch 15/50, Train Loss: 31.8463, Val Loss: 34.1669
Epoch 20/50, Train Loss: 30.4339, Val Loss: 29.8353
Epoch 25/50, Train Loss: 29.4598, Val Loss: 29.9372
Epoch 30/50, Train Loss: 28.5810, Val Loss: 30.2653
Epoch 35/50, Train Loss: 28.2348, Val Loss: 30.8683
Epoch 40/50, Train Loss: 29.0202, Val Loss: 29.2636
Epoch 45/50, Train Loss: 28.3602, Val Loss: 27.1610
Epoch 50/50, Train Loss: 27.9718, Val Loss: 27.8682
The function took 71.18 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.5394
Training Precision Scores: [24.43531828 99.17130208]
Training Recall Scores: [83.02325581 88.7804878 ]
Training F1 Scores: [37.75780011 93.68866963]

Validation Accuracy: 88.8629
Validation Precision Scores: [22.02898551 99.23526766]
Validation Recall Scores: [81.72043011 89.13131313]
Validation F1 Scores: [34.70319635 93.91230311]

Threshold: 0.5
Validation Accuracy: 88.8629
Validation Precision Scores: [22.02898551 99.23526766]
Validation Recall Scores: [81.72043011 89.13131313]
Validation F1 Scores: [34.70319635 93.91230311]


Threshold: 0.6
Validation Accuracy: 92.1340
Validation Precision Scores: [27.93522267 98.96596295]
Validation Recall Scores: [74.19354839 92.80808081]
Validation F1 Scores: [40.58823529 95.7881568 ]


Threshold: 0.7
Validation Accuracy: 94.1978
Validation Precision Scores: [34.09090909 98.62040134]
Validation Recall Scores: [64.51612903 95.31313131]
Validation F1 Scores: [44.60966543 96.93856585]


Threshold: 0.8
Validation Accuracy: 96.1059
Validation Precision Scores: [46.46464646 98.0963953 ]
Validation Recall Scores: [49.46236559 97.85858586]
Validation F1 Scores: [47.91666667 97.97734628]


Threshold: 0.9
Validation Accuracy: 96.4953
Validation Precision Scores: [52.94117647 97.37783075]
Validation Recall Scores: [29.03225806 99.03030303]
Validation F1 Scores: [37.5        98.19711538]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 256
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 41.8085, Val Loss: 43.3539
Epoch 10/50, Train Loss: 37.8155, Val Loss: 37.3237
Epoch 15/50, Train Loss: 33.2925, Val Loss: 33.2437
Epoch 20/50, Train Loss: 33.1030, Val Loss: 33.7355
Epoch 25/50, Train Loss: 32.8212, Val Loss: 33.5139
Epoch 30/50, Train Loss: 31.0872, Val Loss: 33.5573
Early stopping triggered after 31 epochs because no improvements over 10 subsequent epochs!
Epoch 31/50, Train Loss: 30.2494, Val Loss: 33.6207
The function took 47.91 seconds to execute.
No description has been provided for this image
Training Accuracy: 83.6806
Training Precision Scores: [18.14928425 99.09790714]
Training Recall Scores: [82.55813953 83.7296748 ]
Training F1 Scores: [29.75691534 90.76787485]

Validation Accuracy: 83.1386
Validation Precision Scores: [15.86345382 99.3236715 ]
Validation Recall Scores: [84.94623656 83.07070707]
Validation F1 Scores: [26.73434856 90.4730473 ]

Threshold: 0.5
Validation Accuracy: 83.1386
Validation Precision Scores: [15.86345382 99.3236715 ]
Validation Recall Scores: [84.94623656 83.07070707]
Validation F1 Scores: [26.73434856 90.4730473 ]


Threshold: 0.6
Validation Accuracy: 88.5125
Validation Precision Scores: [19.94047619 98.83512545]
Validation Recall Scores: [72.04301075 89.13131313]
Validation F1 Scores: [31.23543124 93.73273847]


Threshold: 0.7
Validation Accuracy: 92.7181
Validation Precision Scores: [24.7311828  98.02686818]
Validation Recall Scores: [49.46236559 94.34343434]
Validation F1 Scores: [32.97491039 96.14988676]


Threshold: 0.8
Validation Accuracy: 94.6651
Validation Precision Scores: [28.         97.36628849]
Validation Recall Scores: [30.10752688 97.09090909]
Validation F1 Scores: [29.01554404 97.2284038 ]


Threshold: 0.9
Validation Accuracy: 95.8723
Validation Precision Scores: [30.3030303  96.72583826]
Validation Recall Scores: [10.75268817 99.07070707]
Validation F1 Scores: [15.87301587 97.88423154]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 320
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 36.9133, Val Loss: 35.4967
Epoch 10/50, Train Loss: 34.1310, Val Loss: 35.7951
Epoch 15/50, Train Loss: 32.3173, Val Loss: 37.1734
Epoch 20/50, Train Loss: 30.8698, Val Loss: 32.0957
Epoch 25/50, Train Loss: 29.4846, Val Loss: 30.0476
Epoch 30/50, Train Loss: 28.3141, Val Loss: 35.0944
Epoch 35/50, Train Loss: 27.7885, Val Loss: 28.3369
Epoch 40/50, Train Loss: 27.2158, Val Loss: 29.9063
Early stopping triggered after 42 epochs because no improvements over 10 subsequent epochs!
Epoch 42/50, Train Loss: 26.4782, Val Loss: 27.9826
The function took 68.72 seconds to execute.
No description has been provided for this image
Training Accuracy: 87.4684
Training Precision Scores: [23.0672533  99.27410992]
Training Recall Scores: [85.34883721 87.56097561]
Training F1 Scores: [36.31865413 93.05038069]

Validation Accuracy: 87.8505
Validation Precision Scores: [21.10817942 99.40612152]
Validation Recall Scores: [86.02150538 87.91919192]
Validation F1 Scores: [33.89830508 93.31046312]

Threshold: 0.5
Validation Accuracy: 87.8505
Validation Precision Scores: [21.10817942 99.40612152]
Validation Recall Scores: [86.02150538 87.91919192]
Validation F1 Scores: [33.89830508 93.31046312]


Threshold: 0.6
Validation Accuracy: 91.5888
Validation Precision Scores: [26.07003891 98.87494591]
Validation Recall Scores: [72.04301075 92.32323232]
Validation F1 Scores: [38.28571429 95.48683661]


Threshold: 0.7
Validation Accuracy: 93.9642
Validation Precision Scores: [32.18390805 98.45446951]
Validation Recall Scores: [60.21505376 95.23232323]
Validation F1 Scores: [41.94756554 96.81659478]


Threshold: 0.8
Validation Accuracy: 95.5997
Validation Precision Scores: [40.19607843 97.89132198]
Validation Recall Scores: [44.08602151 97.53535354]
Validation F1 Scores: [42.05128205 97.71301356]


Threshold: 0.9
Validation Accuracy: 96.2227
Validation Precision Scores: [45.83333333 97.18253968]
Validation Recall Scores: [23.65591398 98.94949495]
Validation F1 Scores: [31.20567376 98.05805806]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 384
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 38.8210, Val Loss: 36.2724
Epoch 10/50, Train Loss: 34.6908, Val Loss: 35.7989
Epoch 15/50, Train Loss: 34.0336, Val Loss: 33.9518
Epoch 20/50, Train Loss: 33.6006, Val Loss: 37.7107
Epoch 25/50, Train Loss: 31.2911, Val Loss: 35.0181
Epoch 30/50, Train Loss: 29.9647, Val Loss: 33.0264
Epoch 35/50, Train Loss: 28.2764, Val Loss: 29.0928
Epoch 40/50, Train Loss: 28.5746, Val Loss: 34.4996
Early stopping triggered after 45 epochs because no improvements over 10 subsequent epochs!
Epoch 45/50, Train Loss: 28.8282, Val Loss: 29.3745
The function took 77.47 seconds to execute.
No description has been provided for this image
Training Accuracy: 83.7098
Training Precision Scores: [18.33927662 99.15733719]
Training Recall Scores: [83.72093023 83.70934959]
Training F1 Scores: [30.08775595 90.78084532]

Validation Accuracy: 83.1386
Validation Precision Scores: [16.         99.37137331]
Validation Recall Scores: [86.02150538 83.03030303]
Validation F1 Scores: [26.98145025 90.46885318]

Threshold: 0.5
Validation Accuracy: 83.1386
Validation Precision Scores: [16.         99.37137331]
Validation Recall Scores: [86.02150538 83.03030303]
Validation F1 Scores: [26.98145025 90.46885318]


Threshold: 0.6
Validation Accuracy: 89.4860
Validation Precision Scores: [21.9047619  98.93475366]
Validation Recall Scores: [74.19354839 90.06060606]
Validation F1 Scores: [33.82352941 94.2893401 ]


Threshold: 0.7
Validation Accuracy: 92.7960
Validation Precision Scores: [28.5046729  98.64061172]
Validation Recall Scores: [65.59139785 93.81818182]
Validation F1 Scores: [39.73941368 96.16897908]


Threshold: 0.8
Validation Accuracy: 95.2103
Validation Precision Scores: [37.06896552 97.96084829]
Validation Recall Scores: [46.23655914 97.05050505]
Validation F1 Scores: [41.14832536 97.50355186]


Threshold: 0.9
Validation Accuracy: 96.1449
Validation Precision Scores: [45.58823529 97.52      ]
Validation Recall Scores: [33.33333333 98.50505051]
Validation F1 Scores: [38.50931677 98.01005025]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 512
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 39.7102, Val Loss: 38.5145
Epoch 10/50, Train Loss: 36.0425, Val Loss: 35.0467
Epoch 15/50, Train Loss: 33.7809, Val Loss: 34.0456
Epoch 20/50, Train Loss: 32.1476, Val Loss: 33.4125
Epoch 25/50, Train Loss: 30.7288, Val Loss: 32.3429
Epoch 30/50, Train Loss: 30.3039, Val Loss: 31.8330
Epoch 35/50, Train Loss: 29.6560, Val Loss: 32.0833
Epoch 40/50, Train Loss: 28.5426, Val Loss: 33.0498
Epoch 45/50, Train Loss: 27.5094, Val Loss: 35.7289
Epoch 50/50, Train Loss: 27.4977, Val Loss: 31.9988
The function took 92.57 seconds to execute.
No description has been provided for this image
Training Accuracy: 84.9075
Training Precision Scores: [20.05347594 99.3452381 ]
Training Recall Scores: [87.20930233 84.80691057]
Training F1 Scores: [32.60869565 91.50219298]

Validation Accuracy: 84.2290
Validation Precision Scores: [16.52360515 99.23882017]
Validation Recall Scores: [82.79569892 84.28282828]
Validation F1 Scores: [27.54919499 91.15140922]

Threshold: 0.5
Validation Accuracy: 84.2290
Validation Precision Scores: [16.52360515 99.23882017]
Validation Recall Scores: [82.79569892 84.28282828]
Validation F1 Scores: [27.54919499 91.15140922]


Threshold: 0.6
Validation Accuracy: 89.0187
Validation Precision Scores: [21.62162162 99.06040268]
Validation Recall Scores: [77.41935484 89.45454545]
Validation F1 Scores: [33.8028169  94.01273885]


Threshold: 0.7
Validation Accuracy: 93.1464
Validation Precision Scores: [29.14572864 98.52258337]
Validation Recall Scores: [62.3655914 94.3030303]
Validation F1 Scores: [39.7260274  96.36663914]


Threshold: 0.8
Validation Accuracy: 94.2368
Validation Precision Scores: [29.62962963 97.8216194 ]
Validation Recall Scores: [43.01075269 96.16161616]
Validation F1 Scores: [35.0877193  96.98451508]


Threshold: 0.9
Validation Accuracy: 95.8723
Validation Precision Scores: [40.         97.32321215]
Validation Recall Scores: [27.95698925 98.42424242]
Validation F1 Scores: [32.91139241 97.87063078]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 16
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 37.9200, Val Loss: 36.6346
Epoch 10/50, Train Loss: 36.9178, Val Loss: 33.5208
Epoch 15/50, Train Loss: 36.7597, Val Loss: 33.1844
Epoch 20/50, Train Loss: 34.9843, Val Loss: 34.1674
Epoch 25/50, Train Loss: 33.9700, Val Loss: 33.2079
Epoch 30/50, Train Loss: 34.6220, Val Loss: 33.3061
Epoch 35/50, Train Loss: 33.2025, Val Loss: 34.8476
Epoch 40/50, Train Loss: 33.1363, Val Loss: 31.5165
Epoch 45/50, Train Loss: 33.9778, Val Loss: 33.7626
Epoch 50/50, Train Loss: 32.9925, Val Loss: 31.8226
The function took 70.55 seconds to execute.
No description has been provided for this image
Training Accuracy: 77.7313
Training Precision Scores: [14.38434983 99.28226543]
Training Recall Scores: [87.20930233 77.31707317]
Training F1 Scores: [24.69542311 86.93366851]

Validation Accuracy: 78.2321
Validation Precision Scores: [12.89808917 99.3814433 ]
Validation Recall Scores: [87.09677419 77.8989899 ]
Validation F1 Scores: [22.46879334 87.33861835]

Threshold: 0.5
Validation Accuracy: 78.2321
Validation Precision Scores: [12.89808917 99.3814433 ]
Validation Recall Scores: [87.09677419 77.8989899 ]
Validation F1 Scores: [22.46879334 87.33861835]


Threshold: 0.6
Validation Accuracy: 86.0202
Validation Precision Scores: [17.56097561 99.02687674]
Validation Recall Scores: [77.41935484 86.34343434]
Validation F1 Scores: [28.62823062 92.2512411 ]


Threshold: 0.7
Validation Accuracy: 91.3162
Validation Precision Scores: [24.40944882 98.66032844]
Validation Recall Scores: [66.66666667 92.24242424]
Validation F1 Scores: [35.73487032 95.34349551]


Threshold: 0.8
Validation Accuracy: 94.1199
Validation Precision Scores: [29.57746479 97.89777411]
Validation Recall Scores: [45.16129032 95.95959596]
Validation F1 Scores: [35.74468085 96.91899612]


Threshold: 0.9
Validation Accuracy: 95.8333
Validation Precision Scores: [33.33333333 96.87252573]
Validation Recall Scores: [15.05376344 98.86868687]
Validation F1 Scores: [20.74074074 97.86042791]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 32
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 37.5914, Val Loss: 36.9375
Epoch 10/50, Train Loss: 35.3452, Val Loss: 33.3349
Epoch 15/50, Train Loss: 33.9796, Val Loss: 32.1849
Epoch 20/50, Train Loss: 32.2270, Val Loss: 32.5368
Epoch 25/50, Train Loss: 33.5316, Val Loss: 32.5335
Early stopping triggered after 29 epochs because no improvements over 10 subsequent epochs!
Epoch 29/50, Train Loss: 32.5992, Val Loss: 31.4920
The function took 41.59 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.2804
Training Precision Scores: [19.69040248 98.70595032]
Training Recall Scores: [73.95348837 86.81910569]
Training F1 Scores: [31.1002445  92.38172479]

Validation Accuracy: 86.9159
Validation Precision Scores: [18.60465116 99.03713893]
Validation Recall Scores: [77.41935484 87.27272727]
Validation F1 Scores: [30.         92.78350515]

Threshold: 0.5
Validation Accuracy: 86.9159
Validation Precision Scores: [18.60465116 99.03713893]
Validation Recall Scores: [77.41935484 87.27272727]
Validation F1 Scores: [30.         92.78350515]


Threshold: 0.6
Validation Accuracy: 91.3941
Validation Precision Scores: [23.55371901 98.45227859]
Validation Recall Scores: [61.29032258 92.52525253]
Validation F1 Scores: [34.02985075 95.39679233]


Threshold: 0.7
Validation Accuracy: 94.5872
Validation Precision Scores: [31.74603175 97.82964783]
Validation Recall Scores: [43.01075269 96.52525253]
Validation F1 Scores: [36.52968037 97.17307301]


Threshold: 0.8
Validation Accuracy: 95.6386
Validation Precision Scores: [36.23188406 97.27891156]
Validation Recall Scores: [26.88172043 98.22222222]
Validation F1 Scores: [30.86419753 97.74829111]


Threshold: 0.9
Validation Accuracy: 96.3396
Validation Precision Scores: [48.14814815 96.85163322]
Validation Recall Scores: [13.97849462 99.43434343]
Validation F1 Scores: [21.66666667 98.12599681]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 64
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 35.5042, Val Loss: 35.9069
Epoch 10/50, Train Loss: 33.5398, Val Loss: 35.0139
Epoch 15/50, Train Loss: 31.4917, Val Loss: 32.9719
Epoch 20/50, Train Loss: 30.3578, Val Loss: 30.2027
Epoch 25/50, Train Loss: 29.3772, Val Loss: 30.8651
Early stopping triggered after 30 epochs because no improvements over 10 subsequent epochs!
Epoch 30/50, Train Loss: 28.8099, Val Loss: 31.9625
The function took 42.55 seconds to execute.
No description has been provided for this image
Training Accuracy: 84.7420
Training Precision Scores: [19.96830428 99.37925272]
Training Recall Scores: [87.90697674 84.60365854]
Training F1 Scores: [32.54412398 91.39814459]

Validation Accuracy: 84.3458
Validation Precision Scores: [17.0575693  99.38065746]
Validation Recall Scores: [86.02150538 84.28282828]
Validation F1 Scores: [28.46975089 91.2111937 ]

Threshold: 0.5
Validation Accuracy: 84.3458
Validation Precision Scores: [17.0575693  99.38065746]
Validation Recall Scores: [86.02150538 84.28282828]
Validation F1 Scores: [28.46975089 91.2111937 ]


Threshold: 0.6
Validation Accuracy: 88.9019
Validation Precision Scores: [21.42857143 99.05913978]
Validation Recall Scores: [77.41935484 89.33333333]
Validation F1 Scores: [33.56643357 93.94518802]


Threshold: 0.7
Validation Accuracy: 92.5234
Validation Precision Scores: [27.18894009 98.55380689]
Validation Recall Scores: [63.44086022 93.61616162]
Validation F1 Scores: [38.06451613 96.02154994]


Threshold: 0.8
Validation Accuracy: 94.5093
Validation Precision Scores: [32.85714286 98.06425041]
Validation Recall Scores: [49.46236559 96.2020202 ]
Validation F1 Scores: [39.48497854 97.12420967]


Threshold: 0.9
Validation Accuracy: 96.3785
Validation Precision Scores: [50.         97.56389776]
Validation Recall Scores: [34.40860215 98.70707071]
Validation F1 Scores: [40.76433121 98.13215505]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 128
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 35.5609, Val Loss: 35.6651
Epoch 10/50, Train Loss: 32.0268, Val Loss: 34.0040
Epoch 15/50, Train Loss: 29.9121, Val Loss: 29.0933
Epoch 20/50, Train Loss: 28.1188, Val Loss: 27.1686
Epoch 25/50, Train Loss: 25.8208, Val Loss: 27.7698
Epoch 30/50, Train Loss: 24.7274, Val Loss: 27.7432
Epoch 35/50, Train Loss: 26.0880, Val Loss: 26.7918
Epoch 40/50, Train Loss: 23.8051, Val Loss: 25.9526
Early stopping triggered after 42 epochs because no improvements over 10 subsequent epochs!
Epoch 42/50, Train Loss: 23.7856, Val Loss: 27.0014
The function took 61.28 seconds to execute.
No description has been provided for this image
Training Accuracy: 89.8734
Training Precision Scores: [27.37388724 99.31629679]
Training Recall Scores: [85.81395349 90.05081301]
Training F1 Scores: [41.50731159 94.45688093]

Validation Accuracy: 90.9268
Validation Precision Scores: [26.82119205 99.47043248]
Validation Recall Scores: [87.09677419 91.07070707]
Validation F1 Scores: [41.01265823 95.08542502]

Threshold: 0.5
Validation Accuracy: 90.9268
Validation Precision Scores: [26.82119205 99.47043248]
Validation Recall Scores: [87.09677419 91.07070707]
Validation F1 Scores: [41.01265823 95.08542502]


Threshold: 0.6
Validation Accuracy: 93.3801
Validation Precision Scores: [32.25806452 99.0216929 ]
Validation Recall Scores: [75.2688172  94.06060606]
Validation F1 Scores: [45.16129032 96.47741401]


Threshold: 0.7
Validation Accuracy: 95.0935
Validation Precision Scores: [39.07284768 98.59329748]
Validation Recall Scores: [63.44086022 96.28282828]
Validation F1 Scores: [48.36065574 97.42436631]


Threshold: 0.8
Validation Accuracy: 95.8333
Validation Precision Scores: [43.39622642 98.09098294]
Validation Recall Scores: [49.46236559 97.57575758]
Validation F1 Scores: [46.23115578 97.83269192]


Threshold: 0.9
Validation Accuracy: 96.5732
Validation Precision Scores: [53.96825397 97.64471058]
Validation Recall Scores: [36.55913978 98.82828283]
Validation F1 Scores: [43.58974359 98.23293173]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 256
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 36.0009, Val Loss: 41.8602
Epoch 10/50, Train Loss: 33.0101, Val Loss: 31.5693
Epoch 15/50, Train Loss: 29.2946, Val Loss: 28.6842
Epoch 20/50, Train Loss: 28.0412, Val Loss: 28.0675
Epoch 25/50, Train Loss: 26.3218, Val Loss: 31.5809
Epoch 30/50, Train Loss: 24.9840, Val Loss: 31.0275
Epoch 35/50, Train Loss: 25.1489, Val Loss: 26.9191
Early stopping triggered after 39 epochs because no improvements over 10 subsequent epochs!
Epoch 39/50, Train Loss: 26.4979, Val Loss: 26.4603
The function took 61.82 seconds to execute.
No description has been provided for this image
Training Accuracy: 84.7225
Training Precision Scores: [20.04208311 99.41450591]
Training Recall Scores: [88.60465116 84.55284553]
Training F1 Scores: [32.68983269 91.38338184]

Validation Accuracy: 83.4502
Validation Precision Scores: [16.93227092 99.61277832]
Validation Recall Scores: [91.39784946 83.15151515]
Validation F1 Scores: [28.57142857 90.64082801]

Threshold: 0.5
Validation Accuracy: 83.4502
Validation Precision Scores: [16.93227092 99.61277832]
Validation Recall Scores: [91.39784946 83.15151515]
Validation F1 Scores: [28.57142857 90.64082801]


Threshold: 0.6
Validation Accuracy: 88.8629
Validation Precision Scores: [22.96918768 99.50248756]
Validation Recall Scores: [88.17204301 88.88888889]
Validation F1 Scores: [36.44444444 93.89671362]


Threshold: 0.7
Validation Accuracy: 92.5234
Validation Precision Scores: [30.58823529 99.35149157]
Validation Recall Scores: [83.87096774 92.84848485]
Validation F1 Scores: [44.82758621 95.98997494]


Threshold: 0.8
Validation Accuracy: 94.6262
Validation Precision Scores: [38.0952381  99.11727617]
Validation Recall Scores: [77.41935484 95.27272727]
Validation F1 Scores: [51.06382979 97.15698393]


Threshold: 0.9
Validation Accuracy: 96.2617
Validation Precision Scores: [48.8        98.69013508]
Validation Recall Scores: [65.59139785 97.41414141]
Validation F1 Scores: [55.96330275 98.04798699]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 320
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 36.3107, Val Loss: 36.0038
Epoch 10/50, Train Loss: 30.5308, Val Loss: 34.2500
Epoch 15/50, Train Loss: 29.9332, Val Loss: 30.1224
Epoch 20/50, Train Loss: 28.6690, Val Loss: 31.5205
Epoch 25/50, Train Loss: 27.1884, Val Loss: 32.4599
Epoch 30/50, Train Loss: 27.0552, Val Loss: 31.5213
Early stopping triggered after 34 epochs because no improvements over 10 subsequent epochs!
Epoch 34/50, Train Loss: 26.0408, Val Loss: 28.4373
The function took 57.91 seconds to execute.
No description has been provided for this image
Training Accuracy: 87.9455
Training Precision Scores: [23.59477124 99.21052632]
Training Recall Scores: [83.95348837 88.1199187 ]
Training F1 Scores: [36.83673469 93.33692142]

Validation Accuracy: 87.9673
Validation Precision Scores: [20.96774194 99.31693989]
Validation Recall Scores: [83.87096774 88.12121212]
Validation F1 Scores: [33.5483871  93.38471419]

Threshold: 0.5
Validation Accuracy: 87.9673
Validation Precision Scores: [20.96774194 99.31693989]
Validation Recall Scores: [83.87096774 88.12121212]
Validation F1 Scores: [33.5483871  93.38471419]


Threshold: 0.6
Validation Accuracy: 92.4065
Validation Precision Scores: [28.38983051 98.88507719]
Validation Recall Scores: [72.04301075 93.17171717]
Validation F1 Scores: [40.72948328 95.94341585]


Threshold: 0.7
Validation Accuracy: 94.1978
Validation Precision Scores: [33.13253012 98.41798501]
Validation Recall Scores: [59.13978495 95.51515152]
Validation F1 Scores: [42.47104247 96.94484314]


Threshold: 0.8
Validation Accuracy: 95.3660
Validation Precision Scores: [37.25490196 97.76966748]
Validation Recall Scores: [40.86021505 97.41414141]
Validation F1 Scores: [38.97435897 97.59158065]


Threshold: 0.9
Validation Accuracy: 96.3006
Validation Precision Scores: [48.21428571 97.37261146]
Validation Recall Scores: [29.03225806 98.82828283]
Validation F1 Scores: [36.24161074 98.09504712]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 384
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 35.4706, Val Loss: 39.4470
Epoch 10/50, Train Loss: 31.8939, Val Loss: 29.7429
Epoch 15/50, Train Loss: 28.6640, Val Loss: 25.0956
Epoch 20/50, Train Loss: 25.3407, Val Loss: 28.3385
Early stopping triggered after 25 epochs because no improvements over 10 subsequent epochs!
Epoch 25/50, Train Loss: 25.6033, Val Loss: 26.8250
The function took 45.01 seconds to execute.
No description has been provided for this image
Training Accuracy: 87.3515
Training Precision Scores: [22.79273638 99.23901764]
Training Recall Scores: [84.65116279 87.4695122 ]
Training F1 Scores: [35.91514554 92.98330903]

Validation Accuracy: 87.6947
Validation Precision Scores: [20.42440318 99.26973984]
Validation Recall Scores: [82.79569892 87.87878788]
Validation F1 Scores: [32.76595745 93.22760394]

Threshold: 0.5
Validation Accuracy: 87.6947
Validation Precision Scores: [20.42440318 99.26973984]
Validation Recall Scores: [82.79569892 87.87878788]
Validation F1 Scores: [32.76595745 93.22760394]


Threshold: 0.6
Validation Accuracy: 91.4720
Validation Precision Scores: [27.3381295  99.25764192]
Validation Recall Scores: [81.72043011 91.83838384]
Validation F1 Scores: [40.9703504  95.40398741]


Threshold: 0.7
Validation Accuracy: 94.0031
Validation Precision Scores: [34.19689119 98.86315789]
Validation Recall Scores: [70.96774194 94.86868687]
Validation F1 Scores: [46.15384615 96.82474227]


Threshold: 0.8
Validation Accuracy: 95.7165
Validation Precision Scores: [42.60869565 98.20627803]
Validation Recall Scores: [52.68817204 97.33333333]
Validation F1 Scores: [47.11538462 97.76785714]


Threshold: 0.9
Validation Accuracy: 96.4174
Validation Precision Scores: [50.74626866 97.64094362]
Validation Recall Scores: [36.55913978 98.66666667]
Validation F1 Scores: [42.5       98.1511254]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 512
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 37.0629, Val Loss: 33.0329
Epoch 10/50, Train Loss: 30.8857, Val Loss: 30.0271
Epoch 15/50, Train Loss: 29.0917, Val Loss: 28.1318
Epoch 20/50, Train Loss: 29.9781, Val Loss: 26.4795
Epoch 25/50, Train Loss: 27.4135, Val Loss: 37.4790
Epoch 30/50, Train Loss: 26.3547, Val Loss: 29.3664
Epoch 35/50, Train Loss: 24.5625, Val Loss: 25.1090
Epoch 40/50, Train Loss: 24.3150, Val Loss: 23.0054
Epoch 45/50, Train Loss: 24.7122, Val Loss: 27.3990
Early stopping triggered after 50 epochs because no improvements over 10 subsequent epochs!
Epoch 50/50, Train Loss: 23.3842, Val Loss: 27.9558
The function took 97.56 seconds to execute.
No description has been provided for this image
Training Accuracy: 89.1237
Training Precision Scores: [26.29399586 99.44450743]
Training Recall Scores: [88.60465116 89.14634146]
Training F1 Scores: [40.5534859  94.01425433]

Validation Accuracy: 88.5514
Validation Precision Scores: [22.46575342 99.50068089]
Validation Recall Scores: [88.17204301 88.56565657]
Validation F1 Scores: [35.80786026 93.71526293]

Threshold: 0.5
Validation Accuracy: 88.5514
Validation Precision Scores: [22.46575342 99.50068089]
Validation Recall Scores: [88.17204301 88.56565657]
Validation F1 Scores: [35.80786026 93.71526293]


Threshold: 0.6
Validation Accuracy: 91.9393
Validation Precision Scores: [28.57142857 99.26151173]
Validation Recall Scores: [81.72043011 92.32323232]
Validation F1 Scores: [42.33983287 95.66673645]


Threshold: 0.7
Validation Accuracy: 94.1589
Validation Precision Scores: [35.23316062 98.94736842]
Validation Recall Scores: [73.11827957 94.94949495]
Validation F1 Scores: [47.55244755 96.90721649]


Threshold: 0.8
Validation Accuracy: 95.5607
Validation Precision Scores: [42.10526316 98.48049281]
Validation Recall Scores: [60.21505376 96.88888889]
Validation F1 Scores: [49.55752212 97.67820774]


Threshold: 0.9
Validation Accuracy: 96.1449
Validation Precision Scores: [46.875      98.05825243]
Validation Recall Scores: [48.38709677 97.93939394]
Validation F1 Scores: [47.61904762 97.99878714]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 32
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 36.4349, Val Loss: 32.5019
Epoch 10/50, Train Loss: 31.7313, Val Loss: 35.7801
Epoch 15/50, Train Loss: 31.1630, Val Loss: 32.6056
Epoch 20/50, Train Loss: 30.6161, Val Loss: 30.7822
Epoch 25/50, Train Loss: 30.0166, Val Loss: 31.5883
Epoch 30/50, Train Loss: 28.6403, Val Loss: 31.3023
Epoch 35/50, Train Loss: 28.4866, Val Loss: 31.9818
Epoch 40/50, Train Loss: 28.2116, Val Loss: 29.4441
Early stopping triggered after 42 epochs because no improvements over 10 subsequent epochs!
Epoch 42/50, Train Loss: 28.5507, Val Loss: 31.9648
The function took 63.52 seconds to execute.
No description has been provided for this image
Training Accuracy: 82.0448
Training Precision Scores: [17.86363636 99.54151177]
Training Recall Scores: [91.39534884 81.63617886]
Training F1 Scores: [29.88593156 89.70407594]

Validation Accuracy: 81.6199
Validation Precision Scores: [14.84230056 99.35929029]
Validation Recall Scores: [86.02150538 81.45454545]
Validation F1 Scores: [25.3164557  89.52042629]

Threshold: 0.5
Validation Accuracy: 81.6199
Validation Precision Scores: [14.84230056 99.35929029]
Validation Recall Scores: [86.02150538 81.45454545]
Validation F1 Scores: [25.3164557  89.52042629]


Threshold: 0.6
Validation Accuracy: 87.7726
Validation Precision Scores: [20.05420054 99.1359709 ]
Validation Recall Scores: [79.56989247 88.08080808]
Validation F1 Scores: [32.03463203 93.28198545]


Threshold: 0.7
Validation Accuracy: 92.5623
Validation Precision Scores: [28.31858407 98.7617421 ]
Validation Recall Scores: [68.8172043  93.45454545]
Validation F1 Scores: [40.12539185 96.03487648]


Threshold: 0.8
Validation Accuracy: 94.5093
Validation Precision Scores: [33.56164384 98.18331957]
Validation Recall Scores: [52.68817204 96.08080808]
Validation F1 Scores: [41.0041841  97.12068613]


Threshold: 0.9
Validation Accuracy: 95.7555
Validation Precision Scores: [38.88888889 97.39583333]
Validation Recall Scores: [30.10752688 98.22222222]
Validation F1 Scores: [33.93939394 97.80728224]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 64
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 34.4116, Val Loss: 34.1422
Epoch 10/50, Train Loss: 31.0055, Val Loss: 31.1288
Epoch 15/50, Train Loss: 30.2672, Val Loss: 30.4491
Epoch 20/50, Train Loss: 28.3933, Val Loss: 28.0525
Epoch 25/50, Train Loss: 28.0069, Val Loss: 32.2816
Early stopping triggered after 30 epochs because no improvements over 10 subsequent epochs!
Epoch 30/50, Train Loss: 27.5881, Val Loss: 28.7767
The function took 45.11 seconds to execute.
No description has been provided for this image
Training Accuracy: 87.6436
Training Precision Scores: [22.81270253 99.10622207]
Training Recall Scores: [81.86046512 87.89634146]
Training F1 Scores: [35.68170299 93.16529326]

Validation Accuracy: 87.5779
Validation Precision Scores: [19.78609626 99.13400182]
Validation Recall Scores: [79.56989247 87.87878788]
Validation F1 Scores: [31.69164882 93.16770186]

Threshold: 0.5
Validation Accuracy: 87.5779
Validation Precision Scores: [19.78609626 99.13400182]
Validation Recall Scores: [79.56989247 87.87878788]
Validation F1 Scores: [31.69164882 93.16770186]


Threshold: 0.6
Validation Accuracy: 91.0436
Validation Precision Scores: [23.55212355 98.61411867]
Validation Recall Scores: [65.59139785 92.        ]
Validation F1 Scores: [34.65909091 95.19230769]


Threshold: 0.7
Validation Accuracy: 94.7819
Validation Precision Scores: [36.05442177 98.34779017]
Validation Recall Scores: [56.98924731 96.2020202 ]
Validation F1 Scores: [44.16666667 97.2630719 ]


Threshold: 0.8
Validation Accuracy: 96.1838
Validation Precision Scores: [47.12643678 97.90407094]
Validation Recall Scores: [44.08602151 98.14141414]
Validation F1 Scores: [45.55555556 98.02259887]


Threshold: 0.9
Validation Accuracy: 96.6121
Validation Precision Scores: [58.33333333 97.1563981 ]
Validation Recall Scores: [22.58064516 99.39393939]
Validation F1 Scores: [32.55813953 98.26243259]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 128
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 34.0942, Val Loss: 30.2014
Epoch 10/50, Train Loss: 30.4436, Val Loss: 29.2898
Epoch 15/50, Train Loss: 27.6922, Val Loss: 30.9038
Epoch 20/50, Train Loss: 26.7772, Val Loss: 27.7113
Epoch 25/50, Train Loss: 25.4282, Val Loss: 27.2100
Epoch 30/50, Train Loss: 24.9330, Val Loss: 27.0306
Epoch 35/50, Train Loss: 23.4029, Val Loss: 32.4968
Epoch 40/50, Train Loss: 22.4740, Val Loss: 26.0224
Epoch 45/50, Train Loss: 22.2819, Val Loss: 31.8307
Epoch 50/50, Train Loss: 22.2843, Val Loss: 28.0718
The function took 76.20 seconds to execute.
No description has been provided for this image
Training Accuracy: 85.4333
Training Precision Scores: [21.46680942 99.65484408]
Training Recall Scores: [93.25581395 85.09146341]
Training F1 Scores: [34.89991297 91.79914483]

Validation Accuracy: 85.5140
Validation Precision Scores: [18.36734694 99.43582511]
Validation Recall Scores: [87.09677419 85.45454545]
Validation F1 Scores: [30.33707865 91.91655802]

Threshold: 0.5
Validation Accuracy: 85.5140
Validation Precision Scores: [18.36734694 99.43582511]
Validation Recall Scores: [87.09677419 85.45454545]
Validation F1 Scores: [30.33707865 91.91655802]


Threshold: 0.6
Validation Accuracy: 89.0187
Validation Precision Scores: [22.76657061 99.36965331]
Validation Recall Scores: [84.94623656 89.17171717]
Validation F1 Scores: [35.90909091 93.99488927]


Threshold: 0.7
Validation Accuracy: 91.9003
Validation Precision Scores: [28.62453532 99.30404524]
Validation Recall Scores: [82.79569892 92.24242424]
Validation F1 Scores: [42.54143646 95.64306661]


Threshold: 0.8
Validation Accuracy: 94.2757
Validation Precision Scores: [35.16483516 98.7845767 ]
Validation Recall Scores: [68.8172043  95.23232323]
Validation F1 Scores: [46.54545455 96.97593088]


Threshold: 0.9
Validation Accuracy: 96.0670
Validation Precision Scores: [46.66666667 98.48856209]
Validation Recall Scores: [60.21505376 97.41414141]
Validation F1 Scores: [52.58215962 97.94840544]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 256
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 35.6669, Val Loss: 44.4312
Epoch 10/50, Train Loss: 31.5893, Val Loss: 31.2708
Epoch 15/50, Train Loss: 28.8944, Val Loss: 31.8371
Epoch 20/50, Train Loss: 25.7686, Val Loss: 30.8845
Epoch 25/50, Train Loss: 24.6262, Val Loss: 25.2845
Early stopping triggered after 29 epochs because no improvements over 10 subsequent epochs!
Epoch 29/50, Train Loss: 25.4325, Val Loss: 27.1847
The function took 47.95 seconds to execute.
No description has been provided for this image
Training Accuracy: 84.8880
Training Precision Scores: [20.38014784 99.47468959]
Training Recall Scores: [89.76744186 84.67479675]
Training F1 Scores: [33.21858864 91.48001757]

Validation Accuracy: 84.6573
Validation Precision Scores: [17.77301927 99.52403617]
Validation Recall Scores: [89.24731183 84.48484848]
Validation F1 Scores: [29.64285714 91.38986014]

Threshold: 0.5
Validation Accuracy: 84.6573
Validation Precision Scores: [17.77301927 99.52403617]
Validation Recall Scores: [89.24731183 84.48484848]
Validation F1 Scores: [29.64285714 91.38986014]


Threshold: 0.6
Validation Accuracy: 90.4206
Validation Precision Scores: [25.24271845 99.33598938]
Validation Recall Scores: [83.87096774 90.66666667]
Validation F1 Scores: [38.80597015 94.8035488 ]


Threshold: 0.7
Validation Accuracy: 94.3536
Validation Precision Scores: [37.12871287 99.23922232]
Validation Recall Scores: [80.64516129 94.86868687]
Validation F1 Scores: [50.84745763 97.00475108]


Threshold: 0.8
Validation Accuracy: 95.9112
Validation Precision Scores: [45.08196721 98.44644317]
Validation Recall Scores: [59.13978495 97.29292929]
Validation F1 Scores: [51.1627907  97.86628734]


Threshold: 0.9
Validation Accuracy: 96.9237
Validation Precision Scores: [61.66666667 97.76714514]
Validation Recall Scores: [39.78494624 99.07070707]
Validation F1 Scores: [48.36601307 98.41460967]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 320
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 36.0812, Val Loss: 37.8928
Epoch 10/50, Train Loss: 30.2141, Val Loss: 30.0322
Epoch 15/50, Train Loss: 28.2051, Val Loss: 25.2592
Epoch 20/50, Train Loss: 26.0859, Val Loss: 33.7236
Epoch 25/50, Train Loss: 23.1307, Val Loss: 24.2374
Epoch 30/50, Train Loss: 22.8347, Val Loss: 25.2409
Epoch 35/50, Train Loss: 21.3108, Val Loss: 26.8158
Early stopping triggered after 37 epochs because no improvements over 10 subsequent epochs!
Epoch 37/50, Train Loss: 22.0978, Val Loss: 27.1195
The function took 64.19 seconds to execute.
No description has been provided for this image
Training Accuracy: 87.5560
Training Precision Scores: [23.95577396 99.53714418]
Training Recall Scores: [90.69767442 87.41869919]
Training F1 Scores: [37.90087464 93.08516394]

Validation Accuracy: 87.5779
Validation Precision Scores: [21.46464646 99.63167587]
Validation Recall Scores: [91.39784946 87.43434343]
Validation F1 Scores: [34.76482618 93.13535614]

Threshold: 0.5
Validation Accuracy: 87.5779
Validation Precision Scores: [21.46464646 99.63167587]
Validation Recall Scores: [91.39784946 87.43434343]
Validation F1 Scores: [34.76482618 93.13535614]


Threshold: 0.6
Validation Accuracy: 91.5888
Validation Precision Scores: [28.71972318 99.56121106]
Validation Recall Scores: [89.24731183 91.67676768]
Validation F1 Scores: [43.45549738 95.45645772]


Threshold: 0.7
Validation Accuracy: 94.0421
Validation Precision Scores: [35.71428571 99.23664122]
Validation Recall Scores: [80.64516129 94.54545455]
Validation F1 Scores: [49.5049505  96.83426443]


Threshold: 0.8
Validation Accuracy: 95.9112
Validation Precision Scores: [46.05263158 99.04801325]
Validation Recall Scores: [75.2688172  96.68686869]
Validation F1 Scores: [57.14285714 97.85319975]


Threshold: 0.9
Validation Accuracy: 97.2352
Validation Precision Scores: [63.0952381  98.38969404]
Validation Recall Scores: [56.98924731 98.74747475]
Validation F1 Scores: [59.88700565 98.56825973]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 384
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 35.9466, Val Loss: 36.9110
Epoch 10/50, Train Loss: 29.9527, Val Loss: 28.3663
Epoch 15/50, Train Loss: 28.3227, Val Loss: 27.7759
Epoch 20/50, Train Loss: 26.2488, Val Loss: 26.8327
Epoch 25/50, Train Loss: 23.9363, Val Loss: 35.4073
Epoch 30/50, Train Loss: 25.0184, Val Loss: 28.4684
Epoch 35/50, Train Loss: 23.5111, Val Loss: 27.3258
Early stopping triggered after 39 epochs because no improvements over 10 subsequent epochs!
Epoch 39/50, Train Loss: 23.8274, Val Loss: 32.3510
The function took 71.95 seconds to execute.
No description has been provided for this image
Training Accuracy: 89.2210
Training Precision Scores: [26.54192654 99.46754277]
Training Recall Scores: [89.06976744 89.22764228]
Training F1 Scores: [40.89695675 94.06974875]

Validation Accuracy: 89.3692
Validation Precision Scores: [23.98843931 99.549955  ]
Validation Recall Scores: [89.24731183 89.37373737]
Validation F1 Scores: [37.81321185 94.18777943]

Threshold: 0.5
Validation Accuracy: 89.3692
Validation Precision Scores: [23.98843931 99.549955  ]
Validation Recall Scores: [89.24731183 89.37373737]
Validation F1 Scores: [37.81321185 94.18777943]


Threshold: 0.6
Validation Accuracy: 92.0950
Validation Precision Scores: [29.32330827 99.3483927 ]
Validation Recall Scores: [83.87096774 92.4040404 ]
Validation F1 Scores: [43.454039   95.75047101]


Threshold: 0.7
Validation Accuracy: 94.1589
Validation Precision Scores: [36.09756098 99.19593737]
Validation Recall Scores: [79.56989247 94.70707071]
Validation F1 Scores: [49.66442953 96.89954527]


Threshold: 0.8
Validation Accuracy: 95.4050
Validation Precision Scores: [41.00719424 98.5179086 ]
Validation Recall Scores: [61.29032258 96.68686869]
Validation F1 Scores: [49.13793103 97.59380098]


Threshold: 0.9
Validation Accuracy: 96.5732
Validation Precision Scores: [52.68817204 98.22222222]
Validation Recall Scores: [52.68817204 98.22222222]
Validation F1 Scores: [52.68817204 98.22222222]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 512
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 36.4858, Val Loss: 36.4897
Epoch 10/50, Train Loss: 32.2860, Val Loss: 33.9348
Epoch 15/50, Train Loss: 26.9803, Val Loss: 28.0244
Epoch 20/50, Train Loss: 26.4693, Val Loss: 30.8505
Epoch 25/50, Train Loss: 23.4717, Val Loss: 31.7018
Early stopping triggered after 26 epochs because no improvements over 10 subsequent epochs!
Epoch 26/50, Train Loss: 24.5232, Val Loss: 32.1517
The function took 51.10 seconds to execute.
No description has been provided for this image
Training Accuracy: 85.7936
Training Precision Scores: [21.11173498 99.36388267]
Training Recall Scores: [87.44186047 85.72154472]
Training F1 Scores: [34.01175938 92.03993671]

Validation Accuracy: 85.7866
Validation Precision Scores: [18.66359447 99.43767573]
Validation Recall Scores: [87.09677419 85.73737374]
Validation F1 Scores: [30.74003795 92.08071165]

Threshold: 0.5
Validation Accuracy: 85.7866
Validation Precision Scores: [18.66359447 99.43767573]
Validation Recall Scores: [87.09677419 85.73737374]
Validation F1 Scores: [30.74003795 92.08071165]


Threshold: 0.6
Validation Accuracy: 92.7181
Validation Precision Scores: [29.56521739 98.93071001]
Validation Recall Scores: [73.11827957 93.45454545]
Validation F1 Scores: [42.10526316 96.11468938]


Threshold: 0.7
Validation Accuracy: 95.4050
Validation Precision Scores: [40.74074074 98.43814221]
Validation Recall Scores: [59.13978495 96.76767677]
Validation F1 Scores: [48.24561404 97.59576202]


Threshold: 0.8
Validation Accuracy: 96.2227
Validation Precision Scores: [47.77777778 97.98224374]
Validation Recall Scores: [46.23655914 98.1010101 ]
Validation F1 Scores: [46.99453552 98.04159095]


Threshold: 0.9
Validation Accuracy: 96.3785
Validation Precision Scores: [50.         97.37470167]
Validation Recall Scores: [29.03225806 98.90909091]
Validation F1 Scores: [36.73469388 98.13589898]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 64
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 32.2481, Val Loss: 39.4665
Epoch 10/50, Train Loss: 28.7178, Val Loss: 28.8997
Epoch 15/50, Train Loss: 27.4231, Val Loss: 27.0659
Epoch 20/50, Train Loss: 25.3129, Val Loss: 25.4436
Epoch 25/50, Train Loss: 25.5814, Val Loss: 25.3141
Epoch 30/50, Train Loss: 23.1528, Val Loss: 25.7879
Epoch 35/50, Train Loss: 22.2994, Val Loss: 24.2995
Epoch 40/50, Train Loss: 23.8188, Val Loss: 27.1253
Early stopping triggered after 45 epochs because no improvements over 10 subsequent epochs!
Epoch 45/50, Train Loss: 22.6098, Val Loss: 24.3395
The function took 72.56 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.1986
Training Precision Scores: [25.03192848 99.56341912]
Training Recall Scores: [91.1627907  88.06910569]
Training F1 Scores: [39.27855711 93.46419327]

Validation Accuracy: 88.2009
Validation Precision Scores: [21.77419355 99.45355191]
Validation Recall Scores: [87.09677419 88.24242424]
Validation F1 Scores: [34.83870968 93.51316635]

Threshold: 0.5
Validation Accuracy: 88.2009
Validation Precision Scores: [21.77419355 99.45355191]
Validation Recall Scores: [87.09677419 88.24242424]
Validation F1 Scores: [34.83870968 93.51316635]


Threshold: 0.6
Validation Accuracy: 92.0561
Validation Precision Scores: [28.57142857 99.17713296]
Validation Recall Scores: [79.56989247 92.52525253]
Validation F1 Scores: [42.04545455 95.73578595]


Threshold: 0.7
Validation Accuracy: 94.4704
Validation Precision Scores: [36.89839572 98.99202016]
Validation Recall Scores: [74.19354839 95.23232323]
Validation F1 Scores: [49.28571429 97.07578254]


Threshold: 0.8
Validation Accuracy: 96.4174
Validation Precision Scores: [50.42735043 98.6128111 ]
Validation Recall Scores: [63.44086022 97.65656566]
Validation F1 Scores: [56.19047619 98.13235891]


Threshold: 0.9
Validation Accuracy: 96.6900
Validation Precision Scores: [56.06060606 97.76179057]
Validation Recall Scores: [39.78494624 98.82828283]
Validation F1 Scores: [46.5408805  98.29214386]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 128
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 31.4871, Val Loss: 30.3308
Epoch 10/50, Train Loss: 28.1825, Val Loss: 29.4798
Epoch 15/50, Train Loss: 27.1969, Val Loss: 27.6052
Epoch 20/50, Train Loss: 25.1495, Val Loss: 29.6689
Epoch 25/50, Train Loss: 23.8057, Val Loss: 29.8742
Epoch 30/50, Train Loss: 23.4593, Val Loss: 26.0290
Epoch 35/50, Train Loss: 22.3585, Val Loss: 30.7845
Early stopping triggered after 40 epochs because no improvements over 10 subsequent epochs!
Epoch 40/50, Train Loss: 22.1124, Val Loss: 26.3411
The function took 64.92 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.3486
Training Precision Scores: [22.44897959 99.60028215]
Training Recall Scores: [92.09302326 86.09756098]
Training F1 Scores: [36.09845032 92.3580072 ]

Validation Accuracy: 86.2928
Validation Precision Scores: [19.23990499 99.44108058]
Validation Recall Scores: [87.09677419 86.26262626]
Validation F1 Scores: [31.51750973 92.38424924]

Threshold: 0.5
Validation Accuracy: 86.2928
Validation Precision Scores: [19.23990499 99.44108058]
Validation Recall Scores: [87.09677419 86.26262626]
Validation F1 Scores: [31.51750973 92.38424924]


Threshold: 0.6
Validation Accuracy: 90.8489
Validation Precision Scores: [26.01351351 99.29577465]
Validation Recall Scores: [82.79569892 91.15151515]
Validation F1 Scores: [39.58868895 95.04950495]


Threshold: 0.7
Validation Accuracy: 93.7305
Validation Precision Scores: [34.11214953 99.15038233]
Validation Recall Scores: [78.49462366 94.3030303 ]
Validation F1 Scores: [47.55700326 96.66597639]


Threshold: 0.8
Validation Accuracy: 95.5607
Validation Precision Scores: [42.22222222 98.52034525]
Validation Recall Scores: [61.29032258 96.84848485]
Validation F1 Scores: [50.         97.67726161]


Threshold: 0.9
Validation Accuracy: 96.5732
Validation Precision Scores: [52.94117647 98.06685461]
Validation Recall Scores: [48.38709677 98.38383838]
Validation F1 Scores: [50.56179775 98.22509076]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 256
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 31.5433, Val Loss: 31.1909
Epoch 10/50, Train Loss: 27.9040, Val Loss: 26.5501
Epoch 15/50, Train Loss: 24.9998, Val Loss: 27.7084
Epoch 20/50, Train Loss: 23.4445, Val Loss: 26.7385
Early stopping triggered after 24 epochs because no improvements over 10 subsequent epochs!
Epoch 24/50, Train Loss: 21.2309, Val Loss: 25.3292
The function took 43.24 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.4713
Training Precision Scores: [25.22996058 99.47416552]
Training Recall Scores: [89.30232558 88.43495935]
Training F1 Scores: [39.3442623  93.63029912]

Validation Accuracy: 88.3178
Validation Precision Scores: [21.79836512 99.40935938]
Validation Recall Scores: [86.02150538 88.4040404 ]
Validation F1 Scores: [34.7826087  93.58426005]

Threshold: 0.5
Validation Accuracy: 88.3178
Validation Precision Scores: [21.79836512 99.40935938]
Validation Recall Scores: [86.02150538 88.4040404 ]
Validation F1 Scores: [34.7826087  93.58426005]


Threshold: 0.6
Validation Accuracy: 92.5234
Validation Precision Scores: [30.12048193 99.22380336]
Validation Recall Scores: [80.64516129 92.96969697]
Validation F1 Scores: [43.85964912 95.99499374]


Threshold: 0.7
Validation Accuracy: 94.9766
Validation Precision Scores: [39.53488372 98.95659432]
Validation Recall Scores: [73.11827957 95.7979798 ]
Validation F1 Scores: [51.32075472 97.35167317]


Threshold: 0.8
Validation Accuracy: 95.8723
Validation Precision Scores: [44.34782609 98.28781084]
Validation Recall Scores: [54.83870968 97.41414141]
Validation F1 Scores: [49.03846154 97.84902597]


Threshold: 0.9
Validation Accuracy: 96.2227
Validation Precision Scores: [46.96969697 97.52198241]
Validation Recall Scores: [33.33333333 98.58585859]
Validation F1 Scores: [38.99371069 98.05103476]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 320
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 31.8201, Val Loss: 30.1860
Epoch 10/50, Train Loss: 28.8874, Val Loss: 30.1681
Epoch 15/50, Train Loss: 26.4350, Val Loss: 27.4466
Epoch 20/50, Train Loss: 24.0498, Val Loss: 28.3749
Epoch 25/50, Train Loss: 23.2077, Val Loss: 27.4230
Early stopping triggered after 26 epochs because no improvements over 10 subsequent epochs!
Epoch 26/50, Train Loss: 22.9098, Val Loss: 28.1542
The function took 49.34 seconds to execute.
No description has been provided for this image
Training Accuracy: 89.1237
Training Precision Scores: [25.72438163 99.25465839]
Training Recall Scores: [84.65116279 89.31910569]
Training F1 Scores: [39.45799458 94.02514041]

Validation Accuracy: 88.4346
Validation Precision Scores: [22.2826087 99.5      ]
Validation Recall Scores: [88.17204301 88.44444444]
Validation F1 Scores: [35.57483731 93.64705882]

Threshold: 0.5
Validation Accuracy: 88.4346
Validation Precision Scores: [22.2826087 99.5      ]
Validation Recall Scores: [88.17204301 88.44444444]
Validation F1 Scores: [35.57483731 93.64705882]


Threshold: 0.6
Validation Accuracy: 93.0296
Validation Precision Scores: [30.45454545 98.89267462]
Validation Recall Scores: [72.04301075 93.81818182]
Validation F1 Scores: [42.8115016  96.28861704]


Threshold: 0.7
Validation Accuracy: 95.1713
Validation Precision Scores: [39.73509934 98.63467108]
Validation Recall Scores: [64.51612903 96.32323232]
Validation F1 Scores: [49.18032787 97.46524939]


Threshold: 0.8
Validation Accuracy: 96.2617
Validation Precision Scores: [48.45360825 98.1384055 ]
Validation Recall Scores: [50.53763441 97.97979798]
Validation F1 Scores: [49.47368421 98.05903761]


Threshold: 0.9
Validation Accuracy: 96.6900
Validation Precision Scores: [59.09090909 97.34548336]
Validation Recall Scores: [27.95698925 99.27272727]
Validation F1 Scores: [37.95620438 98.29965993]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 384
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 32.7942, Val Loss: 38.8752
Epoch 10/50, Train Loss: 29.2502, Val Loss: 33.2500
Epoch 15/50, Train Loss: 24.9828, Val Loss: 28.8606
Epoch 20/50, Train Loss: 24.3056, Val Loss: 29.5028
Early stopping triggered after 21 epochs because no improvements over 10 subsequent epochs!
Epoch 21/50, Train Loss: 23.8951, Val Loss: 30.7951
The function took 42.14 seconds to execute.
No description has been provided for this image
Training Accuracy: 84.2064
Training Precision Scores: [19.24664603 99.31589054]
Training Recall Scores: [86.74418605 84.09552846]
Training F1 Scores: [31.50337838 91.07418006]

Validation Accuracy: 84.5405
Validation Precision Scores: [17.38197425 99.42911513]
Validation Recall Scores: [87.09677419 84.44444444]
Validation F1 Scores: [28.980322  91.3261962]

Threshold: 0.5
Validation Accuracy: 84.5405
Validation Precision Scores: [17.38197425 99.42911513]
Validation Recall Scores: [87.09677419 84.44444444]
Validation F1 Scores: [28.980322  91.3261962]


Threshold: 0.6
Validation Accuracy: 89.1745
Validation Precision Scores: [22.05438066 99.10594546]
Validation Recall Scores: [78.49462366 89.57575758]
Validation F1 Scores: [34.43396226 94.10016978]


Threshold: 0.7
Validation Accuracy: 92.2508
Validation Precision Scores: [26.95652174 98.67408041]
Validation Recall Scores: [66.66666667 93.21212121]
Validation F1 Scores: [38.39009288 95.86536464]


Threshold: 0.8
Validation Accuracy: 94.2368
Validation Precision Scores: [31.54362416 98.09838776]
Validation Recall Scores: [50.53763441 95.87878788]
Validation F1 Scores: [38.84297521 96.97588884]


Threshold: 0.9
Validation Accuracy: 95.9112
Validation Precision Scores: [42.85714286 97.70531401]
Validation Recall Scores: [38.70967742 98.06060606]
Validation F1 Scores: [40.6779661  97.88263763]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 512
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 34.8962, Val Loss: 37.0460
Epoch 10/50, Train Loss: 28.3177, Val Loss: 30.9490
Epoch 15/50, Train Loss: 27.4451, Val Loss: 27.0874
Epoch 20/50, Train Loss: 24.4178, Val Loss: 29.4363
Epoch 25/50, Train Loss: 22.9420, Val Loss: 27.8007
Epoch 30/50, Train Loss: 24.4667, Val Loss: 25.1280
Early stopping triggered after 32 epochs because no improvements over 10 subsequent epochs!
Epoch 32/50, Train Loss: 23.1241, Val Loss: 25.6815
The function took 67.02 seconds to execute.
No description has been provided for this image
Training Accuracy: 84.6543
Training Precision Scores: [20.03138075 99.43766451]
Training Recall Scores: [89.06976744 84.46138211]
Training F1 Scores: [32.70708796 91.33970766]

Validation Accuracy: 84.6184
Validation Precision Scores: [17.45689655 99.42965779]
Validation Recall Scores: [87.09677419 84.52525253]
Validation F1 Scores: [29.08438061 91.37366237]

Threshold: 0.5
Validation Accuracy: 84.6184
Validation Precision Scores: [17.45689655 99.42965779]
Validation Recall Scores: [87.09677419 84.52525253]
Validation F1 Scores: [29.08438061 91.37366237]


Threshold: 0.6
Validation Accuracy: 90.2259
Validation Precision Scores: [24.01315789 99.11660777]
Validation Recall Scores: [78.49462366 90.66666667]
Validation F1 Scores: [36.77581864 94.70352395]


Threshold: 0.7
Validation Accuracy: 93.2632
Validation Precision Scores: [30.76923077 98.77118644]
Validation Recall Scores: [68.8172043  94.18181818]
Validation F1 Scores: [42.52491694 96.42192347]


Threshold: 0.8
Validation Accuracy: 95.4050
Validation Precision Scores: [40.87591241 98.4779926 ]
Validation Recall Scores: [60.21505376 96.72727273]
Validation F1 Scores: [48.69565217 97.5947819 ]


Threshold: 0.9
Validation Accuracy: 96.8069
Validation Precision Scores: [56.17977528 98.26542961]
Validation Recall Scores: [53.76344086 98.42424242]
Validation F1 Scores: [54.94505495 98.3447719 ]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 128
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 30.9684, Val Loss: 37.8827
Epoch 10/50, Train Loss: 27.4666, Val Loss: 30.3953
Epoch 15/50, Train Loss: 26.3469, Val Loss: 29.2560
Epoch 20/50, Train Loss: 24.6003, Val Loss: 28.1082
Epoch 25/50, Train Loss: 22.7924, Val Loss: 28.2675
Epoch 30/50, Train Loss: 22.2489, Val Loss: 24.6500
Epoch 35/50, Train Loss: 19.1823, Val Loss: 25.8477
Early stopping triggered after 36 epochs because no improvements over 10 subsequent epochs!
Epoch 36/50, Train Loss: 21.0164, Val Loss: 25.3100
The function took 65.85 seconds to execute.
No description has been provided for this image
Training Accuracy: 90.8374
Training Precision Scores: [30.23975251 99.56555642]
Training Recall Scores: [90.93023256 90.83333333]
Training F1 Scores: [45.38595473 94.99920285]

Validation Accuracy: 90.8879
Validation Precision Scores: [26.57807309 99.42655492]
Validation Recall Scores: [86.02150538 91.07070707]
Validation F1 Scores: [40.60913706 95.06537326]

Threshold: 0.5
Validation Accuracy: 90.8879
Validation Precision Scores: [26.57807309 99.42655492]
Validation Recall Scores: [86.02150538 91.07070707]
Validation F1 Scores: [40.60913706 95.06537326]


Threshold: 0.6
Validation Accuracy: 93.8863
Validation Precision Scores: [35.45454545 99.36115843]
Validation Recall Scores: [83.87096774 94.26262626]
Validation F1 Scores: [49.84025559 96.74476467]


Threshold: 0.7
Validation Accuracy: 95.7165
Validation Precision Scores: [43.97163121 98.72270293]
Validation Recall Scores: [66.66666667 96.80808081]
Validation F1 Scores: [52.99145299 97.75601795]


Threshold: 0.8
Validation Accuracy: 96.6511
Validation Precision Scores: [53.33333333 98.49776695]
Validation Recall Scores: [60.21505376 98.02020202]
Validation F1 Scores: [56.56565657 98.25840421]


Threshold: 0.9
Validation Accuracy: 96.9237
Validation Precision Scores: [60.         97.95836669]
Validation Recall Scores: [45.16129032 98.86868687]
Validation F1 Scores: [51.53374233 98.41142168]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 256
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 31.0657, Val Loss: 34.7423
Epoch 10/50, Train Loss: 30.2400, Val Loss: 30.1931
Epoch 15/50, Train Loss: 29.0654, Val Loss: 30.7214
Epoch 20/50, Train Loss: 24.2088, Val Loss: 27.2114
Epoch 25/50, Train Loss: 22.4656, Val Loss: 29.3814
Epoch 30/50, Train Loss: 20.3861, Val Loss: 33.9755
Early stopping triggered after 34 epochs because no improvements over 10 subsequent epochs!
Epoch 34/50, Train Loss: 21.8881, Val Loss: 31.7635
The function took 66.42 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.4031
Training Precision Scores: [25.08185986 99.46242708]
Training Recall Scores: [89.06976744 88.37398374]
Training F1 Scores: [39.14154318 93.59091643]

Validation Accuracy: 88.3956
Validation Precision Scores: [22.07084469 99.45479328]
Validation Recall Scores: [87.09677419 88.44444444]
Validation F1 Scores: [35.2173913  93.62703165]

Threshold: 0.5
Validation Accuracy: 88.3956
Validation Precision Scores: [22.07084469 99.45479328]
Validation Recall Scores: [87.09677419 88.44444444]
Validation F1 Scores: [35.2173913  93.62703165]


Threshold: 0.6
Validation Accuracy: 92.2118
Validation Precision Scores: [28.68525896 99.09365559]
Validation Recall Scores: [77.41935484 92.76767677]
Validation F1 Scores: [41.86046512 95.8263773 ]


Threshold: 0.7
Validation Accuracy: 94.6262
Validation Precision Scores: [36.19631902 98.58627859]
Validation Recall Scores: [63.44086022 95.7979798 ]
Validation F1 Scores: [46.09375    97.17213115]


Threshold: 0.8
Validation Accuracy: 95.7555
Validation Precision Scores: [43.5483871  98.40425532]
Validation Recall Scores: [58.06451613 97.17171717]
Validation F1 Scores: [49.76958525 97.78410246]


Threshold: 0.9
Validation Accuracy: 96.6121
Validation Precision Scores: [54.54545455 97.72182254]
Validation Recall Scores: [38.70967742 98.78787879]
Validation F1 Scores: [45.28301887 98.25195901]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 320
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 32.0823, Val Loss: 33.7521
Epoch 10/50, Train Loss: 28.7652, Val Loss: 33.3932
Epoch 15/50, Train Loss: 25.1126, Val Loss: 36.3572
Epoch 20/50, Train Loss: 23.3557, Val Loss: 32.0674
Epoch 25/50, Train Loss: 21.2063, Val Loss: 31.1142
Early stopping triggered after 28 epochs because no improvements over 10 subsequent epochs!
Epoch 28/50, Train Loss: 20.4974, Val Loss: 33.7039
The function took 58.09 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.9094
Training Precision Scores: [25.60220234 99.34217988]
Training Recall Scores: [86.51162791 89.01422764]
Training F1 Scores: [39.51141795 93.8950528 ]

Validation Accuracy: 88.9408
Validation Precision Scores: [22.31884058 99.28025191]
Validation Recall Scores: [82.79569892 89.17171717]
Validation F1 Scores: [35.15981735 93.95487441]

Threshold: 0.5
Validation Accuracy: 88.9408
Validation Precision Scores: [22.31884058 99.28025191]
Validation Recall Scores: [82.79569892 89.17171717]
Validation F1 Scores: [35.15981735 93.95487441]


Threshold: 0.6
Validation Accuracy: 92.2897
Validation Precision Scores: [28.91566265 99.09443726]
Validation Recall Scores: [77.41935484 92.84848485]
Validation F1 Scores: [42.10526316 95.8698373 ]


Threshold: 0.7
Validation Accuracy: 94.5093
Validation Precision Scores: [36.36363636 98.78762542]
Validation Recall Scores: [68.8172043  95.47474747]
Validation F1 Scores: [47.58364312 97.10293815]


Threshold: 0.8
Validation Accuracy: 95.5607
Validation Precision Scores: [41.17647059 98.20334831]
Validation Recall Scores: [52.68817204 97.17171717]
Validation F1 Scores: [46.22641509 97.6848091 ]


Threshold: 0.9
Validation Accuracy: 96.6511
Validation Precision Scores: [55.73770492 97.64658955]
Validation Recall Scores: [36.55913978 98.90909091]
Validation F1 Scores: [44.15584416 98.27378563]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 384
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 32.7077, Val Loss: 33.1561
Epoch 10/50, Train Loss: 28.0669, Val Loss: 27.3432
Epoch 15/50, Train Loss: 24.0067, Val Loss: 25.1995
Epoch 20/50, Train Loss: 22.3045, Val Loss: 28.9219
Epoch 25/50, Train Loss: 22.5515, Val Loss: 27.4518
Epoch 30/50, Train Loss: 20.5674, Val Loss: 29.9700
Epoch 35/50, Train Loss: 18.9417, Val Loss: 25.9597
Early stopping triggered after 39 epochs because no improvements over 10 subsequent epochs!
Epoch 39/50, Train Loss: 20.8148, Val Loss: 26.2574
The function took 83.45 seconds to execute.
No description has been provided for this image
Training Accuracy: 91.9085
Training Precision Scores: [32.55004352 99.38603223]
Training Recall Scores: [86.97674419 92.12398374]
Training F1 Scores: [47.37175427 95.61731976]

Validation Accuracy: 91.6277
Validation Precision Scores: [27.73722628 99.25893636]
Validation Recall Scores: [81.72043011 92.        ]
Validation F1 Scores: [41.41689373 95.49171734]

Threshold: 0.5
Validation Accuracy: 91.6277
Validation Precision Scores: [27.73722628 99.25893636]
Validation Recall Scores: [81.72043011 92.        ]
Validation F1 Scores: [41.41689373 95.49171734]


Threshold: 0.6
Validation Accuracy: 94.6262
Validation Precision Scores: [38.21989529 99.15860328]
Validation Recall Scores: [78.49462366 95.23232323]
Validation F1 Scores: [51.4084507  97.15581204]


Threshold: 0.7
Validation Accuracy: 95.9112
Validation Precision Scores: [45.94594595 98.96694215]
Validation Recall Scores: [73.11827957 96.76767677]
Validation F1 Scores: [56.43153527 97.85495403]


Threshold: 0.8
Validation Accuracy: 96.7290
Validation Precision Scores: [54.12844037 98.61732412]
Validation Recall Scores: [63.44086022 97.97979798]
Validation F1 Scores: [58.41584158 98.29752736]


Threshold: 0.9
Validation Accuracy: 97.1963
Validation Precision Scores: [64.38356164 98.15631263]
Validation Recall Scores: [50.53763441 98.94949495]
Validation F1 Scores: [56.62650602 98.55130785]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 512
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 31.7973, Val Loss: 36.4783
Epoch 10/50, Train Loss: 27.1004, Val Loss: 30.5018
Epoch 15/50, Train Loss: 24.9957, Val Loss: 34.0907
Epoch 20/50, Train Loss: 23.2362, Val Loss: 44.2907
Early stopping triggered after 23 epochs because no improvements over 10 subsequent epochs!
Epoch 23/50, Train Loss: 24.0044, Val Loss: 37.5148
The function took 52.31 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.2668
Training Precision Scores: [24.45616348 99.32594539]
Training Recall Scores: [86.27906977 88.35365854]
Training F1 Scores: [38.10991269 93.51906632]

Validation Accuracy: 87.8115
Validation Precision Scores: [20.58823529 99.27073838]
Validation Recall Scores: [82.79569892 88.        ]
Validation F1 Scores: [32.9764454  93.29620904]

Threshold: 0.5
Validation Accuracy: 87.8115
Validation Precision Scores: [20.58823529 99.27073838]
Validation Recall Scores: [82.79569892 88.        ]
Validation F1 Scores: [32.9764454  93.29620904]


Threshold: 0.6
Validation Accuracy: 93.4969
Validation Precision Scores: [32.03883495 98.85690093]
Validation Recall Scores: [70.96774194 94.34343434]
Validation F1 Scores: [44.14715719 96.54744676]


Threshold: 0.7
Validation Accuracy: 95.4050
Validation Precision Scores: [41.00719424 98.5179086 ]
Validation Recall Scores: [61.29032258 96.68686869]
Validation F1 Scores: [49.13793103 97.59380098]


Threshold: 0.8
Validation Accuracy: 96.4564
Validation Precision Scores: [51.42857143 97.71817454]
Validation Recall Scores: [38.70967742 98.62626263]
Validation F1 Scores: [44.17177914 98.17011864]


Threshold: 0.9
Validation Accuracy: 96.7679
Validation Precision Scores: [63.88888889 97.23538705]
Validation Recall Scores: [24.7311828  99.47474747]
Validation F1 Scores: [35.65891473 98.34232075]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 256
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 30.8141, Val Loss: 35.5175
Epoch 10/50, Train Loss: 29.4359, Val Loss: 32.4896
Epoch 15/50, Train Loss: 25.1992, Val Loss: 27.5151
Epoch 20/50, Train Loss: 23.7240, Val Loss: 29.3530
Epoch 25/50, Train Loss: 22.9097, Val Loss: 26.6015
Epoch 30/50, Train Loss: 22.5855, Val Loss: 30.5687
Epoch 35/50, Train Loss: 22.1428, Val Loss: 30.6985
Epoch 40/50, Train Loss: 20.1300, Val Loss: 28.6704
Early stopping triggered after 42 epochs because no improvements over 10 subsequent epochs!
Epoch 42/50, Train Loss: 18.2156, Val Loss: 28.3854
The function took 100.90 seconds to execute.
No description has been provided for this image
Training Accuracy: 91.8500
Training Precision Scores: [32.44176014 99.40730985]
Training Recall Scores: [87.44186047 92.04268293]
Training F1 Scores: [47.32536186 95.58334653]

Validation Accuracy: 91.0047
Validation Precision Scores: [25.53191489 99.08136483]
Validation Recall Scores: [77.41935484 91.51515152]
Validation F1 Scores: [38.4        95.14807813]

Threshold: 0.5
Validation Accuracy: 91.0047
Validation Precision Scores: [25.53191489 99.08136483]
Validation Recall Scores: [77.41935484 91.51515152]
Validation F1 Scores: [38.4        95.14807813]


Threshold: 0.6
Validation Accuracy: 93.6137
Validation Precision Scores: [33.17535545 99.02418328]
Validation Recall Scores: [75.2688172 94.3030303]
Validation F1 Scores: [46.05263158 96.60596026]


Threshold: 0.7
Validation Accuracy: 94.6651
Validation Precision Scores: [36.74698795 98.66777685]
Validation Recall Scores: [65.59139785 95.75757576]
Validation F1 Scores: [47.1042471  97.19089604]


Threshold: 0.8
Validation Accuracy: 95.6776
Validation Precision Scores: [43.28358209 98.5620378 ]
Validation Recall Scores: [62.3655914  96.92929293]
Validation F1 Scores: [51.10132159 97.73884702]


Threshold: 0.9
Validation Accuracy: 96.9237
Validation Precision Scores: [57.77777778 98.34543987]
Validation Recall Scores: [55.91397849 98.46464646]
Validation F1 Scores: [56.83060109 98.40500707]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 320
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 31.4625, Val Loss: 30.0590
Epoch 10/50, Train Loss: 27.3243, Val Loss: 29.2751
Epoch 15/50, Train Loss: 24.6347, Val Loss: 29.9443
Epoch 20/50, Train Loss: 22.1066, Val Loss: 26.0488
Epoch 25/50, Train Loss: 20.2852, Val Loss: 35.0586
Early stopping triggered after 29 epochs because no improvements over 10 subsequent epochs!
Epoch 29/50, Train Loss: 20.3637, Val Loss: 31.4996
The function took 72.44 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.1500
Training Precision Scores: [24.69453376 99.47217441]
Training Recall Scores: [89.30232558 88.0995935 ]
Training F1 Scores: [38.69017632 93.44112099]

Validation Accuracy: 88.2788
Validation Precision Scores: [21.27071823 99.27470535]
Validation Recall Scores: [82.79569892 88.48484848]
Validation F1 Scores: [33.84615385 93.56975005]

Threshold: 0.5
Validation Accuracy: 88.2788
Validation Precision Scores: [21.27071823 99.27470535]
Validation Recall Scores: [82.79569892 88.48484848]
Validation F1 Scores: [33.84615385 93.56975005]


Threshold: 0.6
Validation Accuracy: 91.5498
Validation Precision Scores: [27.53623188 99.2582897 ]
Validation Recall Scores: [81.72043011 91.91919192]
Validation F1 Scores: [41.19241192 95.44787078]


Threshold: 0.7
Validation Accuracy: 93.5748
Validation Precision Scores: [33.33333333 99.10714286]
Validation Recall Scores: [77.41935484 94.18181818]
Validation F1 Scores: [46.60194175 96.58172778]


Threshold: 0.8
Validation Accuracy: 94.9377
Validation Precision Scores: [38.36477987 98.67164799]
Validation Recall Scores: [65.59139785 96.04040404]
Validation F1 Scores: [48.41269841 97.33824734]


Threshold: 0.9
Validation Accuracy: 96.1838
Validation Precision Scores: [47.31182796 98.02020202]
Validation Recall Scores: [47.31182796 98.02020202]
Validation F1 Scores: [47.31182796 98.02020202]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 384
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 32.7715, Val Loss: 30.0201
Epoch 10/50, Train Loss: 27.9554, Val Loss: 25.9073
Epoch 15/50, Train Loss: 22.8058, Val Loss: 26.2922
Epoch 20/50, Train Loss: 20.4925, Val Loss: 27.4988
Early stopping triggered after 22 epochs because no improvements over 10 subsequent epochs!
Epoch 22/50, Train Loss: 21.4532, Val Loss: 27.3474
The function took 59.51 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.0078
Training Precision Scores: [21.56973461 99.43522767]
Training Recall Scores: [88.8372093  85.88414634]
Training F1 Scores: [34.71149478 92.16424014]

Validation Accuracy: 85.4751
Validation Precision Scores: [18.75      99.5754717]
Validation Recall Scores: [90.32258065 85.29292929]
Validation F1 Scores: [31.05360444 91.88248096]

Threshold: 0.5
Validation Accuracy: 85.4751
Validation Precision Scores: [18.75      99.5754717]
Validation Recall Scores: [90.32258065 85.29292929]
Validation F1 Scores: [31.05360444 91.88248096]


Threshold: 0.6
Validation Accuracy: 91.5888
Validation Precision Scores: [27.95698925 99.34469201]
Validation Recall Scores: [83.87096774 91.87878788]
Validation F1 Scores: [41.93548387 95.46599496]


Threshold: 0.7
Validation Accuracy: 94.3925
Validation Precision Scores: [36.64921466 99.03239377]
Validation Recall Scores: [75.2688172  95.11111111]
Validation F1 Scores: [49.29577465 97.03215169]


Threshold: 0.8
Validation Accuracy: 95.8723
Validation Precision Scores: [44.53781513 98.36668028]
Validation Recall Scores: [56.98924731 97.33333333]
Validation F1 Scores: [50.         97.84727864]


Threshold: 0.9
Validation Accuracy: 96.7290
Validation Precision Scores: [57.14285714 97.7245509 ]
Validation Recall Scores: [38.70967742 98.90909091]
Validation F1 Scores: [46.15384615 98.31325301]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 512
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 31.5426, Val Loss: 33.8985
Epoch 10/50, Train Loss: 27.9802, Val Loss: 31.7689
Epoch 15/50, Train Loss: 24.5303, Val Loss: 32.4647
Epoch 20/50, Train Loss: 25.6952, Val Loss: 28.4733
Epoch 25/50, Train Loss: 20.9132, Val Loss: 30.1094
Epoch 30/50, Train Loss: 19.8588, Val Loss: 29.6673
Early stopping triggered after 32 epochs because no improvements over 10 subsequent epochs!
Epoch 32/50, Train Loss: 20.8468, Val Loss: 32.5793
The function took 92.62 seconds to execute.
No description has been provided for this image
Training Accuracy: 90.0487
Training Precision Scores: [27.71084337 99.30664281]
Training Recall Scores: [85.58139535 90.24390244]
Training F1 Scores: [41.86575654 94.55861996]

Validation Accuracy: 90.0701
Validation Precision Scores: [24.36708861 99.28952043]
Validation Recall Scores: [82.79569892 90.34343434]
Validation F1 Scores: [37.65281174 94.60545801]

Threshold: 0.5
Validation Accuracy: 90.0701
Validation Precision Scores: [24.36708861 99.28952043]
Validation Recall Scores: [82.79569892 90.34343434]
Validation F1 Scores: [37.65281174 94.60545801]


Threshold: 0.6
Validation Accuracy: 92.8738
Validation Precision Scores: [30.43478261 99.01625321]
Validation Recall Scores: [75.2688172  93.53535354]
Validation F1 Scores: [43.34365325 96.19779763]


Threshold: 0.7
Validation Accuracy: 94.1589
Validation Precision Scores: [34.25414365 98.7012987 ]
Validation Recall Scores: [66.66666667 95.19191919]
Validation F1 Scores: [45.25547445 96.91484986]


Threshold: 0.8
Validation Accuracy: 95.7165
Validation Precision Scores: [42.73504274 98.24561404]
Validation Recall Scores: [53.76344086 97.29292929]
Validation F1 Scores: [47.61904762 97.76695087]


Threshold: 0.9
Validation Accuracy: 96.3785
Validation Precision Scores: [50.         97.71634615]
Validation Recall Scores: [38.70967742 98.54545455]
Validation F1 Scores: [43.63636364 98.12914906]


In [26]:
# 16 and hidden dim 128 threshold 0.9

embedding_size=16
hidden_dim=128
threshold = 0.9 

# Create embedding layer with the given vocabulary size and embedding size
vocabulary_size = len(vectorizer.vocabulary_) + 1  # Add 1 for out-of-vocabulary
embeddings = nn.Embedding(vocabulary_size, embedding_size)
X_train_graph = GraphDataset(X_train, y_train, embeddings)
X_val_graph = GraphDataset(X_val, y_val, embeddings)
X_test_graph = GraphDataset(X_test, y_test, embeddings)

# Create DataLoader for batching
# NOTICE: This data loader is from torch_geometric.loader!! It is not from torch.utils.data
batch_size = 64
train_loader = torch_geometric.loader.DataLoader(X_train_graph, batch_size=batch_size, shuffle=True)
val_loader = torch_geometric.loader.DataLoader(X_val_graph, batch_size=batch_size, shuffle=False)
test_loader = torch_geometric.loader.DataLoader(X_test_graph, batch_size=batch_size, shuffle=False)

class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train_original), y=y_train_original)  # Computing class weights
class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
num_epochs = 50
model = GCN(input_dim=embedding_size, hidden_dim=hidden_dim, output_dim=2).to(device)
optimizer = optim.AdamW(model.parameters(), lr=0.01, weight_decay=0.01)
training_loop(model, train_loader, val_loader, X_train_graph, X_val_graph, device, optimizer, num_epochs, criterion)

test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model_with_threshold(model,val_loader,device,threshold=threshold)

stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 37.3670, Val Loss: 41.0201
Epoch 10/50, Train Loss: 33.9726, Val Loss: 34.1704
Epoch 15/50, Train Loss: 31.5596, Val Loss: 29.4678
Epoch 20/50, Train Loss: 30.3961, Val Loss: 26.6791
Epoch 25/50, Train Loss: 27.9115, Val Loss: 27.4083
Epoch 30/50, Train Loss: 26.0020, Val Loss: 25.5574
Epoch 35/50, Train Loss: 26.8470, Val Loss: 27.6204
Early stopping triggered after 40 epochs because no improvements over 10 subsequent epochs!
Epoch 40/50, Train Loss: 25.3462, Val Loss: 26.0847
The function took 57.13 seconds to execute.
No description has been provided for this image
Test Accuracy: 96.4953
Test Precision Scores: [52.83018868 97.41550696]
Test Recall Scores: [30.10752688 98.98989899]
Test F1 Scores: [38.35616438 98.19639279]

Out[26]:
<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7a93bae125d0>
No description has been provided for this image
No description has been provided for this image

Graph SAGE¶

In [15]:
embedding_sizes = [8,16,32,64,128,256]
hidden_dims = [8,16,32,64,128,256,320, 384, 512]

for i in range(len(embedding_sizes)):
    for j in range(i,len(hidden_dims)):
        embedding_size=embedding_sizes[i]
        hidden_dim=hidden_dims[j]
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print(f"Trying with embedding size {embedding_size} and hidden dim {hidden_dim}")

        # Create embedding layer with the given vocabulary size and embedding size
        vocabulary_size = len(vectorizer.vocabulary_) + 1  # Add 1 for out-of-vocabulary
        embeddings = nn.Embedding(vocabulary_size, embedding_size)
        X_train_graph = GraphDataset(X_train, y_train, embeddings)
        X_val_graph = GraphDataset(X_val, y_val, embeddings)
        X_test_graph = GraphDataset(X_test, y_test, embeddings)
        
        # Create DataLoader for batching
        # NOTICE: This data loader is from torch_geometric.loader!! It is not from torch.utils.data
        batch_size = 64
        train_loader = torch_geometric.loader.DataLoader(X_train_graph, batch_size=batch_size, shuffle=True)
        val_loader = torch_geometric.loader.DataLoader(X_val_graph, batch_size=batch_size, shuffle=False)
        test_loader = torch_geometric.loader.DataLoader(X_test_graph, batch_size=batch_size, shuffle=False)
        
        class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train_original), y=y_train_original)  # Computing class weights
        class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
        criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
        num_epochs = 50
        model = GraphSAGE(input_dim=embedding_size, hidden_dim=hidden_dim, output_dim=2).to(device)
        optimizer = optim.AdamW(model.parameters(), lr=0.01, weight_decay=0.01)
        training_loop(model, train_loader, val_loader, X_train_graph, X_val_graph, device, optimizer, num_epochs, criterion)
        
        train_accuracy, train_precisionscores, train_recallscores, train_f1scores, _, _ = testing_model(model,train_loader,device)
        val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model(model,val_loader,device)
        #test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model(model,test_loader,device)
        
        stats(train_accuracy, train_precisionscores, train_recallscores, train_f1scores, 'Training')
        stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
        #stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')
        #ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
        #ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')
        
        # Test with different thresholds
        for threshold in [0.5, 0.6, 0.7, 0.8, 0.9]:
            print(f"Threshold: {threshold}")
            val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model_with_threshold(model,val_loader,device,threshold=threshold)
            stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
            print()
-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 8
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 42.4182, Val Loss: 40.8873
Epoch 10/50, Train Loss: 40.8287, Val Loss: 38.8620
Epoch 15/50, Train Loss: 39.1520, Val Loss: 40.7064
Epoch 20/50, Train Loss: 38.8658, Val Loss: 34.9887
Epoch 25/50, Train Loss: 37.7685, Val Loss: 34.5356
Epoch 30/50, Train Loss: 36.5703, Val Loss: 33.0668
Epoch 35/50, Train Loss: 36.5097, Val Loss: 33.4247
Epoch 40/50, Train Loss: 35.6928, Val Loss: 38.1752
Epoch 45/50, Train Loss: 35.5721, Val Loss: 36.1797
Early stopping triggered after 46 epochs because no improvements over 10 subsequent epochs!
Epoch 46/50, Train Loss: 35.4310, Val Loss: 35.4654
The function took 60.77 seconds to execute.
No description has been provided for this image
Training Accuracy: 85.0828
Training Precision Scores: [17.31909846 98.39235788]
Training Recall Scores: [67.90697674 85.83333333]
Training F1 Scores: [27.59924386 91.68475901]

Validation Accuracy: 85.5919
Validation Precision Scores: [15.97051597 98.70430356]
Validation Recall Scores: [69.89247312 86.18181818]
Validation F1 Scores: [26.         92.01898188]

Threshold: 0.5
Validation Accuracy: 85.5919
Validation Precision Scores: [15.97051597 98.70430356]
Validation Recall Scores: [69.89247312 86.18181818]
Validation F1 Scores: [26.         92.01898188]


Threshold: 0.6
Validation Accuracy: 90.6153
Validation Precision Scores: [21.53846154 98.39688042]
Validation Recall Scores: [60.21505376 91.75757576]
Validation F1 Scores: [31.72804533 94.96132135]


Threshold: 0.7
Validation Accuracy: 94.1589
Validation Precision Scores: [30.87248322 98.05704837]
Validation Recall Scores: [49.46236559 95.83838384]
Validation F1 Scores: [38.01652893 96.93502248]


Threshold: 0.8
Validation Accuracy: 96.4564
Validation Precision Scores: [52.38095238 97.18923199]
Validation Recall Scores: [23.65591398 99.19191919]
Validation F1 Scores: [32.59259259 98.18036393]


Threshold: 0.9
Validation Accuracy: 96.6511
Validation Precision Scores: [73.33333333 96.78809244]
Validation Recall Scores: [11.82795699 99.83838384]
Validation F1 Scores: [20.37037037 98.28957836]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 16
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 37.8303, Val Loss: 35.5702
Epoch 10/50, Train Loss: 34.6404, Val Loss: 31.7704
Epoch 15/50, Train Loss: 34.0041, Val Loss: 30.7726
Epoch 20/50, Train Loss: 33.7227, Val Loss: 28.8863
Epoch 25/50, Train Loss: 32.1090, Val Loss: 29.3035
Epoch 30/50, Train Loss: 31.4762, Val Loss: 28.6804
Epoch 35/50, Train Loss: 32.0113, Val Loss: 29.1743
Early stopping triggered after 36 epochs because no improvements over 10 subsequent epochs!
Epoch 36/50, Train Loss: 30.5809, Val Loss: 28.0976
The function took 46.85 seconds to execute.
No description has been provided for this image
Training Accuracy: 85.6475
Training Precision Scores: [19.65116279 98.92397661]
Training Recall Scores: [78.60465116 85.95528455]
Training F1 Scores: [31.44186047 91.98477433]

Validation Accuracy: 85.9034
Validation Precision Scores: [17.89976134 99.16240112]
Validation Recall Scores: [80.64516129 86.1010101 ]
Validation F1 Scores: [29.296875   92.17128028]

Threshold: 0.5
Validation Accuracy: 85.9034
Validation Precision Scores: [17.89976134 99.16240112]
Validation Recall Scores: [80.64516129 86.1010101 ]
Validation F1 Scores: [29.296875   92.17128028]


Threshold: 0.6
Validation Accuracy: 93.1075
Validation Precision Scores: [29.61165049 98.64521592]
Validation Recall Scores: [65.59139785 94.14141414]
Validation F1 Scores: [40.80267559 96.34070705]


Threshold: 0.7
Validation Accuracy: 95.4050
Validation Precision Scores: [39.31623932 98.08241534]
Validation Recall Scores: [49.46236559 97.13131313]
Validation F1 Scores: [43.80952381 97.6045473 ]


Threshold: 0.8
Validation Accuracy: 96.4953
Validation Precision Scores: [52.72727273 97.45324314]
Validation Recall Scores: [31.1827957  98.94949495]
Validation F1 Scores: [39.18918919 98.19566961]


Threshold: 0.9
Validation Accuracy: 96.9237
Validation Precision Scores: [79.16666667 97.09119497]
Validation Recall Scores: [20.43010753 99.7979798 ]
Validation F1 Scores: [32.47863248 98.42598127]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 32
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 37.2210, Val Loss: 33.0439
Epoch 10/50, Train Loss: 32.3825, Val Loss: 30.1625
Epoch 15/50, Train Loss: 30.3157, Val Loss: 25.6824
Epoch 20/50, Train Loss: 28.3542, Val Loss: 26.6730
Epoch 25/50, Train Loss: 27.3200, Val Loss: 23.7659
Epoch 30/50, Train Loss: 25.7724, Val Loss: 25.0714
Epoch 35/50, Train Loss: 25.4951, Val Loss: 22.5241
Epoch 40/50, Train Loss: 25.2975, Val Loss: 24.5805
Epoch 45/50, Train Loss: 25.4734, Val Loss: 22.4624
Epoch 50/50, Train Loss: 25.4526, Val Loss: 23.5583
The function took 66.54 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.6310
Training Precision Scores: [22.57126236 99.50882938]
Training Recall Scores: [90.23255814 86.47357724]
Training F1 Scores: [36.10981852 92.53439182]

Validation Accuracy: 86.6433
Validation Precision Scores: [19.80676329 99.48932219]
Validation Recall Scores: [88.17204301 86.58585859]
Validation F1 Scores: [32.34714004 92.59019227]

Threshold: 0.5
Validation Accuracy: 86.6433
Validation Precision Scores: [19.80676329 99.48932219]
Validation Recall Scores: [88.17204301 86.58585859]
Validation F1 Scores: [32.34714004 92.59019227]


Threshold: 0.6
Validation Accuracy: 91.5498
Validation Precision Scores: [27.53623188 99.2582897 ]
Validation Recall Scores: [81.72043011 91.91919192]
Validation F1 Scores: [41.19241192 95.44787078]


Threshold: 0.7
Validation Accuracy: 93.9252
Validation Precision Scores: [34.01015228 98.90341628]
Validation Recall Scores: [72.04301075 94.74747475]
Validation F1 Scores: [46.20689655 96.78085019]


Threshold: 0.8
Validation Accuracy: 95.8333
Validation Precision Scores: [44.6969697  98.60426929]
Validation Recall Scores: [63.44086022 97.05050505]
Validation F1 Scores: [52.44444444 97.82121767]


Threshold: 0.9
Validation Accuracy: 97.0405
Validation Precision Scores: [61.64383562 98.0761523 ]
Validation Recall Scores: [48.38709677 98.86868687]
Validation F1 Scores: [54.21686747 98.47082495]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 64
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 34.5065, Val Loss: 31.3549
Epoch 10/50, Train Loss: 30.3636, Val Loss: 26.4867
Epoch 15/50, Train Loss: 27.3372, Val Loss: 23.9069
Epoch 20/50, Train Loss: 26.0689, Val Loss: 23.4022
Epoch 25/50, Train Loss: 25.2115, Val Loss: 24.3649
Epoch 30/50, Train Loss: 23.1272, Val Loss: 22.7020
Epoch 35/50, Train Loss: 23.0623, Val Loss: 21.2380
Epoch 40/50, Train Loss: 21.8498, Val Loss: 21.7553
Epoch 45/50, Train Loss: 20.9374, Val Loss: 20.4812
Epoch 50/50, Train Loss: 20.5698, Val Loss: 19.9628
The function took 65.80 seconds to execute.
No description has been provided for this image
Training Accuracy: 90.7205
Training Precision Scores: [30.02291826 99.58709965]
Training Recall Scores: [91.39534884 90.69105691]
Training F1 Scores: [45.19838988 94.93112069]

Validation Accuracy: 91.8224
Validation Precision Scores: [28.41328413 99.30343927]
Validation Recall Scores: [82.79569892 92.16161616]
Validation F1 Scores: [42.30769231 95.59932942]

Threshold: 0.5
Validation Accuracy: 91.8224
Validation Precision Scores: [28.41328413 99.30343927]
Validation Recall Scores: [82.79569892 92.16161616]
Validation F1 Scores: [42.30769231 95.59932942]


Threshold: 0.6
Validation Accuracy: 93.6526
Validation Precision Scores: [33.64485981 99.10790144]
Validation Recall Scores: [77.41935484 94.26262626]
Validation F1 Scores: [46.90553746 96.62455995]


Threshold: 0.7
Validation Accuracy: 95.4439
Validation Precision Scores: [42.10526316 98.79966887]
Validation Recall Scores: [68.8172043  96.44444444]
Validation F1 Scores: [52.24489796 97.60785116]


Threshold: 0.8
Validation Accuracy: 96.9626
Validation Precision Scores: [57.57575758 98.54191981]
Validation Recall Scores: [61.29032258 98.3030303 ]
Validation F1 Scores: [59.375     98.4223301]


Threshold: 0.9
Validation Accuracy: 97.3910
Validation Precision Scores: [70.3125     98.08306709]
Validation Recall Scores: [48.38709677 99.23232323]
Validation F1 Scores: [57.32484076 98.65434826]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 128
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 31.5982, Val Loss: 26.0502
Epoch 10/50, Train Loss: 27.2356, Val Loss: 23.9215
Epoch 15/50, Train Loss: 25.8572, Val Loss: 23.1321
Epoch 20/50, Train Loss: 23.4081, Val Loss: 21.9953
Epoch 25/50, Train Loss: 21.3468, Val Loss: 22.0461
Epoch 30/50, Train Loss: 20.7593, Val Loss: 23.4379
Epoch 35/50, Train Loss: 20.6026, Val Loss: 21.9598
Early stopping triggered after 37 epochs because no improvements over 10 subsequent epochs!
Epoch 37/50, Train Loss: 18.4854, Val Loss: 25.5915
The function took 50.37 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.6758
Training Precision Scores: [26.09262883 99.65663271]
Training Recall Scores: [93.02325581 88.48577236]
Training F1 Scores: [40.75394804 93.73957044]

Validation Accuracy: 88.4735
Validation Precision Scores: [22.03856749 99.41043084]
Validation Recall Scores: [86.02150538 88.56565657]
Validation F1 Scores: [35.0877193  93.67521368]

Threshold: 0.5
Validation Accuracy: 88.4735
Validation Precision Scores: [22.03856749 99.41043084]
Validation Recall Scores: [86.02150538 88.56565657]
Validation F1 Scores: [35.0877193  93.67521368]


Threshold: 0.6
Validation Accuracy: 93.1464
Validation Precision Scores: [32.48945148 99.31359931]
Validation Recall Scores: [82.79569892 93.53535354]
Validation F1 Scores: [46.66666667 96.33791094]


Threshold: 0.7
Validation Accuracy: 95.0545
Validation Precision Scores: [40.4494382  99.12133891]
Validation Recall Scores: [77.41935484 95.71717172]
Validation F1 Scores: [53.13653137 97.38951696]


Threshold: 0.8
Validation Accuracy: 96.6511
Validation Precision Scores: [53.04347826 98.69547493]
Validation Recall Scores: [65.59139785 97.81818182]
Validation F1 Scores: [58.65384615 98.25487013]


Threshold: 0.9
Validation Accuracy: 97.3910
Validation Precision Scores: [69.6969697 98.1215028]
Validation Recall Scores: [49.46236559 99.19191919]
Validation F1 Scores: [57.86163522 98.65380751]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 256
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 36.0655, Val Loss: 32.1638
Epoch 10/50, Train Loss: 30.1694, Val Loss: 28.1263
Epoch 15/50, Train Loss: 28.0407, Val Loss: 25.2822
Epoch 20/50, Train Loss: 24.9453, Val Loss: 23.8011
Epoch 25/50, Train Loss: 22.6054, Val Loss: 23.6676
Epoch 30/50, Train Loss: 23.0338, Val Loss: 24.7605
Early stopping triggered after 31 epochs because no improvements over 10 subsequent epochs!
Epoch 31/50, Train Loss: 21.3550, Val Loss: 23.1818
The function took 47.70 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.0565
Training Precision Scores: [22.01117318 99.5754717 ]
Training Recall Scores: [91.62790698 85.81300813]
Training F1 Scores: [35.4954955  92.18340611]

Validation Accuracy: 85.3583
Validation Precision Scores: [18.05869074 99.38823529]
Validation Recall Scores: [86.02150538 85.33333333]
Validation F1 Scores: [29.85074627 91.82608696]

Threshold: 0.5
Validation Accuracy: 85.3583
Validation Precision Scores: [18.05869074 99.38823529]
Validation Recall Scores: [86.02150538 85.33333333]
Validation F1 Scores: [29.85074627 91.82608696]


Threshold: 0.6
Validation Accuracy: 89.8754
Validation Precision Scores: [23.82445141 99.24410849]
Validation Recall Scores: [81.72043011 90.18181818]
Validation F1 Scores: [36.89320388 94.49618967]


Threshold: 0.7
Validation Accuracy: 93.8084
Validation Precision Scores: [33.5        98.90202703]
Validation Recall Scores: [72.04301075 94.62626263]
Validation F1 Scores: [45.7337884  96.71691101]


Threshold: 0.8
Validation Accuracy: 95.9891
Validation Precision Scores: [45.61403509 98.32925835]
Validation Recall Scores: [55.91397849 97.49494949]
Validation F1 Scores: [50.24154589 97.91032664]


Threshold: 0.9
Validation Accuracy: 97.0794
Validation Precision Scores: [65.         97.84688995]
Validation Recall Scores: [41.93548387 99.15151515]
Validation F1 Scores: [50.98039216 98.4948826 ]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 320
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 38.0492, Val Loss: 39.9160
Epoch 10/50, Train Loss: 34.4964, Val Loss: 33.7861
Epoch 15/50, Train Loss: 32.5805, Val Loss: 31.2247
Epoch 20/50, Train Loss: 29.4139, Val Loss: 31.2502
Epoch 25/50, Train Loss: 30.1032, Val Loss: 31.2633
Early stopping triggered after 29 epochs because no improvements over 10 subsequent epochs!
Epoch 29/50, Train Loss: 27.2673, Val Loss: 30.8472
The function took 46.94 seconds to execute.
No description has been provided for this image
Training Accuracy: 81.9182
Training Precision Scores: [17.1954023  99.30821495]
Training Recall Scores: [86.97674419 81.69715447]
Training F1 Scores: [28.71401152 89.64594369]

Validation Accuracy: 81.3474
Validation Precision Scores: [13.72180451 99.01768173]
Validation Recall Scores: [78.49462366 81.45454545]
Validation F1 Scores: [23.36       89.38151186]

Threshold: 0.5
Validation Accuracy: 81.3474
Validation Precision Scores: [13.72180451 99.01768173]
Validation Recall Scores: [78.49462366 81.45454545]
Validation F1 Scores: [23.36       89.38151186]


Threshold: 0.6
Validation Accuracy: 85.9813
Validation Precision Scores: [17.1990172  98.93567793]
Validation Recall Scores: [75.2688172  86.38383838]
Validation F1 Scores: [28.         92.23468507]


Threshold: 0.7
Validation Accuracy: 90.2259
Validation Precision Scores: [19.61538462 98.18024263]
Validation Recall Scores: [54.83870968 91.55555556]
Validation F1 Scores: [28.89518414 94.75224754]


Threshold: 0.8
Validation Accuracy: 93.4969
Validation Precision Scores: [25.33333333 97.72539289]
Validation Recall Scores: [40.86021505 95.47474747]
Validation F1 Scores: [31.27572016 96.58696096]


Threshold: 0.9
Validation Accuracy: 95.9112
Validation Precision Scores: [33.33333333 96.80094787]
Validation Recall Scores: [12.90322581 99.03030303]
Validation F1 Scores: [18.60465116 97.90293589]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 384
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 33.5160, Val Loss: 29.7610
Epoch 10/50, Train Loss: 27.6643, Val Loss: 27.9568
Epoch 15/50, Train Loss: 25.4291, Val Loss: 25.8682
Epoch 20/50, Train Loss: 22.8433, Val Loss: 22.7760
Epoch 25/50, Train Loss: 21.6136, Val Loss: 25.5551
Early stopping triggered after 30 epochs because no improvements over 10 subsequent epochs!
Epoch 30/50, Train Loss: 20.2564, Val Loss: 28.3856
The function took 51.55 seconds to execute.
No description has been provided for this image
Training Accuracy: 87.9649
Training Precision Scores: [24.74937343 99.59649527]
Training Recall Scores: [91.86046512 87.79471545]
Training F1 Scores: [38.99308983 93.32397105]

Validation Accuracy: 87.2274
Validation Precision Scores: [20.25316456 99.40174873]
Validation Recall Scores: [86.02150538 87.27272727]
Validation F1 Scores: [32.78688525 92.94320138]

Threshold: 0.5
Validation Accuracy: 87.2274
Validation Precision Scores: [20.25316456 99.40174873]
Validation Recall Scores: [86.02150538 87.27272727]
Validation F1 Scores: [32.78688525 92.94320138]


Threshold: 0.6
Validation Accuracy: 90.7321
Validation Precision Scores: [25.08591065 99.1216513 ]
Validation Recall Scores: [78.49462366 91.19191919]
Validation F1 Scores: [38.02083333 94.99158249]


Threshold: 0.7
Validation Accuracy: 93.9642
Validation Precision Scores: [34.18367347 98.90387858]
Validation Recall Scores: [72.04301075 94.78787879]
Validation F1 Scores: [46.36678201 96.80214566]


Threshold: 0.8
Validation Accuracy: 95.4829
Validation Precision Scores: [41.72661871 98.55907781]
Validation Recall Scores: [62.3655914  96.72727273]
Validation F1 Scores: [50.         97.63458401]


Threshold: 0.9
Validation Accuracy: 96.8069
Validation Precision Scores: [57.33333333 97.99438428]
Validation Recall Scores: [46.23655914 98.70707071]
Validation F1 Scores: [51.19047619 98.34943639]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 512
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 34.9152, Val Loss: 33.8550
Epoch 10/50, Train Loss: 30.6803, Val Loss: 31.2540
Epoch 15/50, Train Loss: 27.8084, Val Loss: 30.1717
Epoch 20/50, Train Loss: 26.6109, Val Loss: 31.6220
Epoch 25/50, Train Loss: 25.8288, Val Loss: 34.2404
Epoch 30/50, Train Loss: 24.0120, Val Loss: 29.9978
Epoch 35/50, Train Loss: 22.7645, Val Loss: 30.4257
Epoch 40/50, Train Loss: 23.8147, Val Loss: 31.4199
Early stopping triggered after 41 epochs because no improvements over 10 subsequent epochs!
Epoch 41/50, Train Loss: 25.4356, Val Loss: 30.3844
The function took 74.64 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.4362
Training Precision Scores: [22.34348076 99.51928714]
Training Recall Scores: [90.46511628 86.2601626 ]
Training F1 Scores: [35.83602027 92.4165714 ]

Validation Accuracy: 86.1371
Validation Precision Scores: [18.76484561 99.34792734]
Validation Recall Scores: [84.94623656 86.18181818]
Validation F1 Scores: [30.73929961 92.29770662]

Threshold: 0.5
Validation Accuracy: 86.1371
Validation Precision Scores: [18.76484561 99.34792734]
Validation Recall Scores: [84.94623656 86.18181818]
Validation F1 Scores: [30.73929961 92.29770662]


Threshold: 0.6
Validation Accuracy: 89.9143
Validation Precision Scores: [23.73417722 99.20071048]
Validation Recall Scores: [80.64516129 90.26262626]
Validation F1 Scores: [36.67481663 94.52083774]


Threshold: 0.7
Validation Accuracy: 92.0950
Validation Precision Scores: [28.34645669 99.09248055]
Validation Recall Scores: [77.41935484 92.64646465]
Validation F1 Scores: [41.49855908 95.76111923]


Threshold: 0.8
Validation Accuracy: 94.0810
Validation Precision Scores: [32.94797688 98.49686848]
Validation Recall Scores: [61.29032258 95.31313131]
Validation F1 Scores: [42.85714286 96.8788501 ]


Threshold: 0.9
Validation Accuracy: 95.5218
Validation Precision Scores: [39.62264151 97.9285134 ]
Validation Recall Scores: [45.16129032 97.41414141]
Validation F1 Scores: [42.21105528 97.67065019]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 16
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 36.2762, Val Loss: 31.5580
Epoch 10/50, Train Loss: 32.3643, Val Loss: 29.5733
Epoch 15/50, Train Loss: 29.0215, Val Loss: 26.7883
Epoch 20/50, Train Loss: 28.8836, Val Loss: 26.0797
Epoch 25/50, Train Loss: 28.7228, Val Loss: 28.6052
Epoch 30/50, Train Loss: 28.7548, Val Loss: 26.4808
Epoch 35/50, Train Loss: 27.0264, Val Loss: 24.7451
Epoch 40/50, Train Loss: 25.8911, Val Loss: 30.0987
Epoch 45/50, Train Loss: 25.9988, Val Loss: 26.2168
Early stopping triggered after 46 epochs because no improvements over 10 subsequent epochs!
Epoch 46/50, Train Loss: 25.8815, Val Loss: 24.8446
The function took 61.58 seconds to execute.
No description has been provided for this image
Training Accuracy: 83.6417
Training Precision Scores: [19.27236971 99.53861098]
Training Recall Scores: [91.1627907  83.31300813]
Training F1 Scores: [31.81818182 90.70590839]

Validation Accuracy: 84.0343
Validation Precision Scores: [17.45379877 99.61556944]
Validation Recall Scores: [91.39784946 83.75757576]
Validation F1 Scores: [29.31034483 91.00087796]

Threshold: 0.5
Validation Accuracy: 84.0343
Validation Precision Scores: [17.45379877 99.61556944]
Validation Recall Scores: [91.39784946 83.75757576]
Validation F1 Scores: [29.31034483 91.00087796]


Threshold: 0.6
Validation Accuracy: 90.6542
Validation Precision Scores: [25.74257426 99.33774834]
Validation Recall Scores: [83.87096774 90.90909091]
Validation F1 Scores: [39.39393939 94.93670886]


Threshold: 0.7
Validation Accuracy: 94.5872
Validation Precision Scores: [36.62790698 98.74791319]
Validation Recall Scores: [67.74193548 95.5959596 ]
Validation F1 Scores: [47.54716981 97.14637651]


Threshold: 0.8
Validation Accuracy: 96.0670
Validation Precision Scores: [46.2962963  98.25203252]
Validation Recall Scores: [53.76344086 97.65656566]
Validation F1 Scores: [49.75124378 97.95339412]


Threshold: 0.9
Validation Accuracy: 96.7679
Validation Precision Scores: [59.25925926 97.57358791]
Validation Recall Scores: [34.40860215 99.11111111]
Validation F1 Scores: [43.53741497 98.33633995]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 32
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 31.5558, Val Loss: 29.0795
Epoch 10/50, Train Loss: 28.6552, Val Loss: 26.4281
Epoch 15/50, Train Loss: 25.6056, Val Loss: 25.1203
Epoch 20/50, Train Loss: 25.4525, Val Loss: 22.8609
Epoch 25/50, Train Loss: 25.1398, Val Loss: 22.0270
Epoch 30/50, Train Loss: 22.2103, Val Loss: 22.6185
Epoch 35/50, Train Loss: 23.0401, Val Loss: 21.9560
Epoch 40/50, Train Loss: 23.4443, Val Loss: 23.3337
Epoch 45/50, Train Loss: 22.4256, Val Loss: 22.7557
Epoch 50/50, Train Loss: 22.2630, Val Loss: 21.9501
The function took 66.71 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.8510
Training Precision Scores: [26.18254497 99.57805907]
Training Recall Scores: [91.39534884 88.7398374 ]
Training F1 Scores: [40.70429829 93.84706325]

Validation Accuracy: 88.7072
Validation Precision Scores: [23.01369863 99.59146618]
Validation Recall Scores: [90.32258065 88.64646465]
Validation F1 Scores: [36.68122271 93.80076956]

Threshold: 0.5
Validation Accuracy: 88.7072
Validation Precision Scores: [23.01369863 99.59146618]
Validation Recall Scores: [90.32258065 88.64646465]
Validation F1 Scores: [36.68122271 93.80076956]


Threshold: 0.6
Validation Accuracy: 92.6402
Validation Precision Scores: [31.1023622  99.39498704]
Validation Recall Scores: [84.94623656 92.92929293]
Validation F1 Scores: [45.53314121 96.05345584]


Threshold: 0.7
Validation Accuracy: 95.9112
Validation Precision Scores: [46.20253165 99.17012448]
Validation Recall Scores: [78.49462366 96.56565657]
Validation F1 Scores: [58.16733068 97.85056295]


Threshold: 0.8
Validation Accuracy: 96.8069
Validation Precision Scores: [55.91397849 98.34343434]
Validation Recall Scores: [55.91397849 98.34343434]
Validation F1 Scores: [55.91397849 98.34343434]


Threshold: 0.9
Validation Accuracy: 97.5078
Validation Precision Scores: [74.57627119 98.04703069]
Validation Recall Scores: [47.31182796 99.39393939]
Validation F1 Scores: [57.89473684 98.71589085]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 64
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 30.9248, Val Loss: 32.3322
Epoch 10/50, Train Loss: 25.4581, Val Loss: 24.1710
Epoch 15/50, Train Loss: 22.9427, Val Loss: 22.8787
Epoch 20/50, Train Loss: 21.7010, Val Loss: 21.7503
Epoch 25/50, Train Loss: 21.2944, Val Loss: 20.9064
Epoch 30/50, Train Loss: 20.3780, Val Loss: 21.0414
Early stopping triggered after 35 epochs because no improvements over 10 subsequent epochs!
Epoch 35/50, Train Loss: 20.6454, Val Loss: 28.6037
The function took 47.88 seconds to execute.
No description has been provided for this image
Training Accuracy: 89.1529
Training Precision Scores: [26.63934426 99.54576425]
Training Recall Scores: [90.69767442 89.08536585]
Training F1 Scores: [41.18268215 94.02552826]

Validation Accuracy: 89.9922
Validation Precision Scores: [24.84662577 99.4647636 ]
Validation Recall Scores: [87.09677419 90.1010101 ]
Validation F1 Scores: [38.66348449 94.55162179]

Threshold: 0.5
Validation Accuracy: 89.9922
Validation Precision Scores: [24.84662577 99.4647636 ]
Validation Recall Scores: [87.09677419 90.1010101 ]
Validation F1 Scores: [38.66348449 94.55162179]


Threshold: 0.6
Validation Accuracy: 93.4579
Validation Precision Scores: [33.9055794  99.40042827]
Validation Recall Scores: [84.94623656 93.77777778]
Validation F1 Scores: [48.46625767 96.50727651]


Threshold: 0.7
Validation Accuracy: 95.6776
Validation Precision Scores: [44.57831325 99.20899251]
Validation Recall Scores: [79.56989247 96.28282828]
Validation F1 Scores: [57.14285714 97.72401066]


Threshold: 0.8
Validation Accuracy: 96.7679
Validation Precision Scores: [54.03225806 98.93617021]
Validation Recall Scores: [72.04301075 97.6969697 ]
Validation F1 Scores: [61.75115207 98.31266518]


Threshold: 0.9
Validation Accuracy: 97.1184
Validation Precision Scores: [61.44578313 98.30985915]
Validation Recall Scores: [54.83870968 98.70707071]
Validation F1 Scores: [57.95454545 98.50806452]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 128
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 33.9144, Val Loss: 33.1917
Epoch 10/50, Train Loss: 26.1333, Val Loss: 25.6123
Epoch 15/50, Train Loss: 22.6714, Val Loss: 23.1422
Epoch 20/50, Train Loss: 21.4640, Val Loss: 24.3631
Epoch 25/50, Train Loss: 19.5920, Val Loss: 22.8986
Early stopping triggered after 29 epochs because no improvements over 10 subsequent epochs!
Epoch 29/50, Train Loss: 20.6041, Val Loss: 24.9230
The function took 41.47 seconds to execute.
No description has been provided for this image
Training Accuracy: 89.0750
Training Precision Scores: [26.58998647 99.57916288]
Training Recall Scores: [91.39534884 88.97357724]
Training F1 Scores: [41.19496855 93.97810219]

Validation Accuracy: 88.4735
Validation Precision Scores: [21.56862745 99.27634555]
Validation Recall Scores: [82.79569892 88.68686869]
Validation F1 Scores: [34.22222222 93.68331199]

Threshold: 0.5
Validation Accuracy: 88.4735
Validation Precision Scores: [21.56862745 99.27634555]
Validation Recall Scores: [82.79569892 88.68686869]
Validation F1 Scores: [34.22222222 93.68331199]


Threshold: 0.6
Validation Accuracy: 91.5109
Validation Precision Scores: [26.93726937 99.12929909]
Validation Recall Scores: [78.49462366 92.        ]
Validation F1 Scores: [40.10989011 95.43168483]


Threshold: 0.7
Validation Accuracy: 94.1978
Validation Precision Scores: [35.10638298 98.86554622]
Validation Recall Scores: [70.96774194 95.07070707]
Validation F1 Scores: [46.97508897 96.93099897]


Threshold: 0.8
Validation Accuracy: 96.0280
Validation Precision Scores: [46.5648855  98.68691014]
Validation Recall Scores: [65.59139785 97.17171717]
Validation F1 Scores: [54.46428571 97.92345277]


Threshold: 0.9
Validation Accuracy: 96.7290
Validation Precision Scores: [55.05617978 98.22509076]
Validation Recall Scores: [52.68817204 98.38383838]
Validation F1 Scores: [53.84615385 98.30440048]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 256
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 31.8027, Val Loss: 28.1338
Epoch 10/50, Train Loss: 25.0108, Val Loss: 23.4783
Epoch 15/50, Train Loss: 20.3087, Val Loss: 20.1943
Epoch 20/50, Train Loss: 20.8769, Val Loss: 47.0581
Epoch 25/50, Train Loss: 17.6215, Val Loss: 19.2830
Epoch 30/50, Train Loss: 17.8003, Val Loss: 21.9305
Epoch 35/50, Train Loss: 16.4759, Val Loss: 19.9718
Epoch 40/50, Train Loss: 14.9338, Val Loss: 20.9749
Early stopping triggered after 41 epochs because no improvements over 10 subsequent epochs!
Epoch 41/50, Train Loss: 14.5505, Val Loss: 23.5578
The function took 62.89 seconds to execute.
No description has been provided for this image
Training Accuracy: 92.8432
Training Precision Scores: [36.22402891 99.68350977]
Training Recall Scores: [93.25581395 92.82520325]
Training F1 Scores: [52.17957059 96.13218965]

Validation Accuracy: 92.7181
Validation Precision Scores: [31.0483871  99.31034483]
Validation Recall Scores: [82.79569892 93.09090909]
Validation F1 Scores: [45.16129032 96.10010428]

Threshold: 0.5
Validation Accuracy: 92.7181
Validation Precision Scores: [31.0483871  99.31034483]
Validation Recall Scores: [82.79569892 93.09090909]
Validation F1 Scores: [45.16129032 96.10010428]


Threshold: 0.6
Validation Accuracy: 94.8209
Validation Precision Scores: [39.13043478 99.11912752]
Validation Recall Scores: [77.41935484 95.47474747]
Validation F1 Scores: [51.98555957 97.26281128]


Threshold: 0.7
Validation Accuracy: 96.2227
Validation Precision Scores: [48.61111111 99.05115512]
Validation Recall Scores: [75.2688172  97.01010101]
Validation F1 Scores: [59.07172996 98.02000408]


Threshold: 0.8
Validation Accuracy: 96.8847
Validation Precision Scores: [55.37190083 98.93747446]
Validation Recall Scores: [72.04301075 97.81818182]
Validation F1 Scores: [62.61682243 98.37464445]


Threshold: 0.9
Validation Accuracy: 97.4688
Validation Precision Scores: [67.94871795 98.3935743 ]
Validation Recall Scores: [56.98924731 98.98989899]
Validation F1 Scores: [61.98830409 98.69083585]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 320
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 28.3723, Val Loss: 38.2858
Epoch 10/50, Train Loss: 23.2788, Val Loss: 23.0850
Epoch 15/50, Train Loss: 20.6338, Val Loss: 20.2360
Epoch 20/50, Train Loss: 20.0716, Val Loss: 20.4951
Epoch 25/50, Train Loss: 17.9836, Val Loss: 21.7204
Epoch 30/50, Train Loss: 17.8814, Val Loss: 24.0799
Epoch 35/50, Train Loss: 15.6311, Val Loss: 23.8789
Epoch 40/50, Train Loss: 16.9494, Val Loss: 28.7198
Epoch 45/50, Train Loss: 15.6311, Val Loss: 31.0077
Early stopping triggered after 47 epochs because no improvements over 10 subsequent epochs!
Epoch 47/50, Train Loss: 14.4090, Val Loss: 21.2372
The function took 77.36 seconds to execute.
No description has been provided for this image
Training Accuracy: 92.2493
Training Precision Scores: [34.49152542 99.7469747 ]
Training Recall Scores: [94.65116279 92.14430894]
Training F1 Scores: [50.55900621 95.79503434]

Validation Accuracy: 91.7056
Validation Precision Scores: [28.41726619 99.38864629]
Validation Recall Scores: [84.94623656 91.95959596]
Validation F1 Scores: [42.58760108 95.52990556]

Threshold: 0.5
Validation Accuracy: 91.7056
Validation Precision Scores: [28.41726619 99.38864629]
Validation Recall Scores: [84.94623656 91.95959596]
Validation F1 Scores: [42.58760108 95.52990556]


Threshold: 0.6
Validation Accuracy: 94.5872
Validation Precision Scores: [38.26530612 99.24114671]
Validation Recall Scores: [80.64516129 95.11111111]
Validation F1 Scores: [51.90311419 97.13224675]


Threshold: 0.7
Validation Accuracy: 96.0670
Validation Precision Scores: [47.33333333 99.09015715]
Validation Recall Scores: [76.34408602 96.80808081]
Validation F1 Scores: [58.43621399 97.93582669]


Threshold: 0.8
Validation Accuracy: 96.6900
Validation Precision Scores: [53.22580645 98.89525368]
Validation Recall Scores: [70.96774194 97.65656566]
Validation F1 Scores: [60.82949309 98.27200651]


Threshold: 0.9
Validation Accuracy: 97.1184
Validation Precision Scores: [60.6741573  98.42678499]
Validation Recall Scores: [58.06451613 98.58585859]
Validation F1 Scores: [59.34065934 98.50625757]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 384
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 30.5363, Val Loss: 25.1447
Epoch 10/50, Train Loss: 24.6430, Val Loss: 22.7104
Epoch 15/50, Train Loss: 21.5560, Val Loss: 24.0029
Epoch 20/50, Train Loss: 19.5405, Val Loss: 21.8971
Epoch 25/50, Train Loss: 17.8450, Val Loss: 21.4440
Epoch 30/50, Train Loss: 16.3136, Val Loss: 21.3680
Epoch 35/50, Train Loss: 17.3979, Val Loss: 26.1066
Early stopping triggered after 38 epochs because no improvements over 10 subsequent epochs!
Epoch 38/50, Train Loss: 16.0644, Val Loss: 24.7448
The function took 64.93 seconds to execute.
No description has been provided for this image
Training Accuracy: 90.3797
Training Precision Scores: [29.6350365  99.73033708]
Training Recall Scores: [94.41860465 90.20325203]
Training F1 Scores: [45.11111111 94.72785486]

Validation Accuracy: 89.9143
Validation Precision Scores: [24.69512195 99.46428571]
Validation Recall Scores: [87.09677419 90.02020202]
Validation F1 Scores: [38.47980998 94.5068929 ]

Threshold: 0.5
Validation Accuracy: 89.9143
Validation Precision Scores: [24.69512195 99.46428571]
Validation Recall Scores: [87.09677419 90.02020202]
Validation F1 Scores: [38.47980998 94.5068929 ]


Threshold: 0.6
Validation Accuracy: 92.7570
Validation Precision Scores: [31.17408907 99.31064196]
Validation Recall Scores: [82.79569892 93.13131313]
Validation F1 Scores: [45.29411765 96.12176814]


Threshold: 0.7
Validation Accuracy: 94.7819
Validation Precision Scores: [39.03743316 99.1600168 ]
Validation Recall Scores: [78.49462366 95.39393939]
Validation F1 Scores: [52.14285714 97.24052718]


Threshold: 0.8
Validation Accuracy: 95.8333
Validation Precision Scores: [45.         98.76441516]
Validation Recall Scores: [67.74193548 96.88888889]
Validation F1 Scores: [54.07725322 97.81766266]


Threshold: 0.9
Validation Accuracy: 96.6900
Validation Precision Scores: [54.08163265 98.3805668 ]
Validation Recall Scores: [56.98924731 98.18181818]
Validation F1 Scores: [55.4973822  98.28109201]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 512
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 39.0428, Val Loss: 39.4749
Epoch 10/50, Train Loss: 33.9119, Val Loss: 34.2339
Epoch 15/50, Train Loss: 30.9134, Val Loss: 31.4996
Epoch 20/50, Train Loss: 29.0024, Val Loss: 36.1539
Epoch 25/50, Train Loss: 27.7994, Val Loss: 34.0702
Early stopping triggered after 29 epochs because no improvements over 10 subsequent epochs!
Epoch 29/50, Train Loss: 26.2424, Val Loss: 36.4408
The function took 54.22 seconds to execute.
No description has been provided for this image
Training Accuracy: 80.7400
Training Precision Scores: [16.05263158 99.19899875]
Training Recall Scores: [85.11627907 80.54878049]
Training F1 Scores: [27.01107011 88.90633763]

Validation Accuracy: 79.7897
Validation Precision Scores: [13.14878893 99.14572864]
Validation Recall Scores: [81.72043011 79.71717172]
Validation F1 Scores: [22.65275708 88.3762598 ]

Threshold: 0.5
Validation Accuracy: 79.7897
Validation Precision Scores: [13.14878893 99.14572864]
Validation Recall Scores: [81.72043011 79.71717172]
Validation F1 Scores: [22.65275708 88.3762598 ]


Threshold: 0.6
Validation Accuracy: 83.9953
Validation Precision Scores: [14.82300885 98.77126654]
Validation Recall Scores: [72.04301075 84.44444444]
Validation F1 Scores: [24.58715596 91.04770203]


Threshold: 0.7
Validation Accuracy: 89.6417
Validation Precision Scores: [20.87542088 98.63496257]
Validation Recall Scores: [66.66666667 90.50505051]
Validation F1 Scores: [31.79487179 94.39528024]


Threshold: 0.8
Validation Accuracy: 93.1854
Validation Precision Scores: [26.1627907 97.9966611]
Validation Recall Scores: [48.38709677 94.86868687]
Validation F1 Scores: [33.96226415 96.40730856]


Threshold: 0.9
/usr/local/lib/python3.11/dist-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
Validation Accuracy: 96.3785
Validation Precision Scores: [ 0.         96.37850467]
Validation Recall Scores: [  0. 100.]
Validation F1 Scores: [ 0.         98.15585961]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 32
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 30.0831, Val Loss: 25.9861
Epoch 10/50, Train Loss: 25.5547, Val Loss: 22.5183
Epoch 15/50, Train Loss: 24.0023, Val Loss: 20.1427
Epoch 20/50, Train Loss: 23.6512, Val Loss: 20.1142
Epoch 25/50, Train Loss: 21.6874, Val Loss: 19.8884
Epoch 30/50, Train Loss: 22.4624, Val Loss: 20.1421
Epoch 35/50, Train Loss: 20.8581, Val Loss: 18.8428
Epoch 40/50, Train Loss: 19.7004, Val Loss: 19.6339
Early stopping triggered after 45 epochs because no improvements over 10 subsequent epochs!
Epoch 45/50, Train Loss: 20.6610, Val Loss: 19.4815
The function took 63.17 seconds to execute.
No description has been provided for this image
Training Accuracy: 92.9601
Training Precision Scores: [36.24413146 99.52199891]
Training Recall Scores: [89.76744186 93.0995935 ]
Training F1 Scores: [51.63879599 96.20372801]

Validation Accuracy: 93.2632
Validation Precision Scores: [33.33333333 99.44158076]
Validation Recall Scores: [86.02150538 93.53535354]
Validation F1 Scores: [48.04804805 96.39808453]

Threshold: 0.5
Validation Accuracy: 93.2632
Validation Precision Scores: [33.33333333 99.44158076]
Validation Recall Scores: [86.02150538 93.53535354]
Validation F1 Scores: [48.04804805 96.39808453]


Threshold: 0.6
Validation Accuracy: 95.3660
Validation Precision Scores: [42.44186047 99.16527546]
Validation Recall Scores: [78.49462366 96.        ]
Validation F1 Scores: [55.09433962 97.55696982]


Threshold: 0.7
Validation Accuracy: 96.4564
Validation Precision Scores: [50.78125    98.85245902]
Validation Recall Scores: [69.89247312 97.45454545]
Validation F1 Scores: [58.82352941 98.14852492]


Threshold: 0.8
Validation Accuracy: 97.0016
Validation Precision Scores: [58.33333333 98.50323625]
Validation Recall Scores: [60.21505376 98.38383838]
Validation F1 Scores: [59.25925926 98.44350111]


Threshold: 0.9
Validation Accuracy: 97.5467
Validation Precision Scores: [76.78571429 98.00955414]
Validation Recall Scores: [46.23655914 99.47474747]
Validation F1 Scores: [57.71812081 98.73671546]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 64
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 28.2175, Val Loss: 26.4341
Epoch 10/50, Train Loss: 24.4861, Val Loss: 23.0793
Epoch 15/50, Train Loss: 21.7148, Val Loss: 23.8319
Early stopping triggered after 16 epochs because no improvements over 10 subsequent epochs!
Epoch 16/50, Train Loss: 21.8489, Val Loss: 22.8323
The function took 22.74 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.8413
Training Precision Scores: [24.82419128 99.12974684]
Training Recall Scores: [82.09302326 89.13617886]
Training F1 Scores: [38.12095032 93.8677226 ]

Validation Accuracy: 88.4346
Validation Precision Scores: [21.82320442 99.36536718]
Validation Recall Scores: [84.94623656 88.56565657]
Validation F1 Scores: [34.72527473 93.65520188]

Threshold: 0.5
Validation Accuracy: 88.4346
Validation Precision Scores: [21.82320442 99.36536718]
Validation Recall Scores: [84.94623656 88.56565657]
Validation F1 Scores: [34.72527473 93.65520188]


Threshold: 0.6
Validation Accuracy: 91.1994
Validation Precision Scores: [26.16487455 99.12625601]
Validation Recall Scores: [78.49462366 91.67676768]
Validation F1 Scores: [39.24731183 95.25608732]


Threshold: 0.7
Validation Accuracy: 94.3925
Validation Precision Scores: [36.50793651 98.99117276]
Validation Recall Scores: [74.19354839 95.15151515]
Validation F1 Scores: [48.93617021 97.03337454]


Threshold: 0.8
Validation Accuracy: 96.0670
Validation Precision Scores: [46.07843137 98.13463098]
Validation Recall Scores: [50.53763441 97.77777778]
Validation F1 Scores: [48.20512821 97.95587938]


Threshold: 0.9
Validation Accuracy: 96.7290
Validation Precision Scores: [57.14285714 97.7245509 ]
Validation Recall Scores: [38.70967742 98.90909091]
Validation F1 Scores: [46.15384615 98.31325301]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 128
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 26.9850, Val Loss: 23.8013
Epoch 10/50, Train Loss: 22.6596, Val Loss: 21.0199
Epoch 15/50, Train Loss: 20.3139, Val Loss: 21.0062
Early stopping triggered after 17 epochs because no improvements over 10 subsequent epochs!
Epoch 17/50, Train Loss: 18.2938, Val Loss: 21.5227
The function took 24.57 seconds to execute.
No description has been provided for this image
Training Accuracy: 92.1130
Training Precision Scores: [32.9443447  99.31192661]
Training Recall Scores: [85.34883721 92.40853659]
Training F1 Scores: [47.5388601  95.73594441]

Validation Accuracy: 92.5234
Validation Precision Scores: [30.88803089 99.43698571]
Validation Recall Scores: [86.02150538 92.76767677]
Validation F1 Scores: [45.45454545 95.98662207]

Threshold: 0.5
Validation Accuracy: 92.5234
Validation Precision Scores: [30.88803089 99.43698571]
Validation Recall Scores: [86.02150538 92.76767677]
Validation F1 Scores: [45.45454545 95.98662207]


Threshold: 0.6
Validation Accuracy: 95.1324
Validation Precision Scores: [40.58823529 98.99916597]
Validation Recall Scores: [74.19354839 95.91919192]
Validation F1 Scores: [52.47148289 97.43484506]


Threshold: 0.7
Validation Accuracy: 96.3006
Validation Precision Scores: [49.15254237 98.57142857]
Validation Recall Scores: [62.3655914  97.57575758]
Validation F1 Scores: [54.97630332 98.07106599]


Threshold: 0.8
Validation Accuracy: 97.3131
Validation Precision Scores: [65.         98.35209003]
Validation Recall Scores: [55.91397849 98.86868687]
Validation F1 Scores: [60.11560694 98.60971187]


Threshold: 0.9
Validation Accuracy: 97.1573
Validation Precision Scores: [70.83333333 97.65873016]
Validation Recall Scores: [36.55913978 99.43434343]
Validation F1 Scores: [48.22695035 98.53853854]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 256
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 28.9043, Val Loss: 25.8407
Epoch 10/50, Train Loss: 21.9696, Val Loss: 26.8077
Epoch 15/50, Train Loss: 21.9414, Val Loss: 25.6349
Epoch 20/50, Train Loss: 20.3947, Val Loss: 29.2541
Epoch 25/50, Train Loss: 18.8309, Val Loss: 26.3692
Epoch 30/50, Train Loss: 17.3322, Val Loss: 27.7264
Early stopping triggered after 32 epochs because no improvements over 10 subsequent epochs!
Epoch 32/50, Train Loss: 16.1444, Val Loss: 31.1067
The function took 51.11 seconds to execute.
No description has been provided for this image
Training Accuracy: 91.2853
Training Precision Scores: [31.35525261 99.56777125]
Training Recall Scores: [90.93023256 91.30081301]
Training F1 Scores: [46.63088849 95.25526162]

Validation Accuracy: 90.6542
Validation Precision Scores: [25.58139535 99.29422144]
Validation Recall Scores: [82.79569892 90.94949495]
Validation F1 Scores: [39.08629442 94.93884437]

Threshold: 0.5
Validation Accuracy: 90.6542
Validation Precision Scores: [25.58139535 99.29422144]
Validation Recall Scores: [82.79569892 90.94949495]
Validation F1 Scores: [39.08629442 94.93884437]


Threshold: 0.6
Validation Accuracy: 93.6916
Validation Precision Scores: [33.6492891 99.0666101]
Validation Recall Scores: [76.34408602 94.34343434]
Validation F1 Scores: [46.71052632 96.64735099]


Threshold: 0.7
Validation Accuracy: 95.6776
Validation Precision Scores: [44.         98.88337469]
Validation Recall Scores: [70.96774194 96.60606061]
Validation F1 Scores: [54.32098765 97.7314531 ]


Threshold: 0.8
Validation Accuracy: 97.2741
Validation Precision Scores: [61.61616162 98.70392872]
Validation Recall Scores: [65.59139785 98.46464646]
Validation F1 Scores: [63.54166667 98.58414239]


Threshold: 0.9
Validation Accuracy: 97.7414
Validation Precision Scores: [76.11940299 98.32067173]
Validation Recall Scores: [54.83870968 99.35353535]
Validation F1 Scores: [63.75       98.83440514]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 320
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 28.1831, Val Loss: 24.0955
Epoch 10/50, Train Loss: 24.6502, Val Loss: 26.1780
Epoch 15/50, Train Loss: 23.0936, Val Loss: 24.2280
Epoch 20/50, Train Loss: 20.8321, Val Loss: 24.5903
Epoch 25/50, Train Loss: 19.1420, Val Loss: 25.6780
Early stopping triggered after 28 epochs because no improvements over 10 subsequent epochs!
Epoch 28/50, Train Loss: 17.1670, Val Loss: 25.6399
The function took 46.95 seconds to execute.
No description has been provided for this image
Training Accuracy: 92.6680
Training Precision Scores: [35.1425943 99.477295 ]
Training Recall Scores: [88.8372093  92.83536585]
Training F1 Scores: [50.36255768 96.04163381]

Validation Accuracy: 92.1729
Validation Precision Scores: [29.06976744 99.22077922]
Validation Recall Scores: [80.64516129 92.60606061]
Validation F1 Scores: [42.73504274 95.79937304]

Threshold: 0.5
Validation Accuracy: 92.1729
Validation Precision Scores: [29.06976744 99.22077922]
Validation Recall Scores: [80.64516129 92.60606061]
Validation F1 Scores: [42.73504274 95.79937304]


Threshold: 0.6
Validation Accuracy: 94.0031
Validation Precision Scores: [34.35897436 98.9043405 ]
Validation Recall Scores: [72.04301075 94.82828283]
Validation F1 Scores: [46.52777778 96.82343234]


Threshold: 0.7
Validation Accuracy: 95.8333
Validation Precision Scores: [45.         98.76441516]
Validation Recall Scores: [67.74193548 96.88888889]
Validation F1 Scores: [54.07725322 97.81766266]


Threshold: 0.8
Validation Accuracy: 96.7679
Validation Precision Scores: [55.68181818 98.22580645]
Validation Recall Scores: [52.68817204 98.42424242]
Validation F1 Scores: [54.14364641 98.32492432]


Threshold: 0.9
Validation Accuracy: 97.5857
Validation Precision Scores: [76.27118644 98.08688721]
Validation Recall Scores: [48.38709677 99.43434343]
Validation F1 Scores: [59.21052632 98.75601926]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 384
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 28.5593, Val Loss: 25.8037
Epoch 10/50, Train Loss: 21.4137, Val Loss: 21.3057
Epoch 15/50, Train Loss: 19.7933, Val Loss: 22.2284
Epoch 20/50, Train Loss: 17.5159, Val Loss: 28.3911
Early stopping triggered after 24 epochs because no improvements over 10 subsequent epochs!
Epoch 24/50, Train Loss: 17.2128, Val Loss: 22.8734
The function took 42.37 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.0331
Training Precision Scores: [25.17091361 99.71134973]
Training Recall Scores: [94.18604651 87.76422764]
Training F1 Scores: [39.72535557 93.35711583]

Validation Accuracy: 87.9283
Validation Precision Scores: [21.37203166 99.45180448]
Validation Recall Scores: [87.09677419 87.95959596]
Validation F1 Scores: [34.3220339  93.35334477]

Threshold: 0.5
Validation Accuracy: 87.9283
Validation Precision Scores: [21.37203166 99.45180448]
Validation Recall Scores: [87.09677419 87.95959596]
Validation F1 Scores: [34.3220339  93.35334477]


Threshold: 0.6
Validation Accuracy: 91.3551
Validation Precision Scores: [27.20848057 99.29978118]
Validation Recall Scores: [82.79569892 91.67676768]
Validation F1 Scores: [40.95744681 95.33613445]


Threshold: 0.7
Validation Accuracy: 94.0031
Validation Precision Scores: [35.26570048 99.15290131]
Validation Recall Scores: [78.49462366 94.58585859]
Validation F1 Scores: [48.66666667 96.81555004]


Threshold: 0.8
Validation Accuracy: 96.6900
Validation Precision Scores: [53.17460317 98.93529894]
Validation Recall Scores: [72.04301075 97.61616162]
Validation F1 Scores: [61.18721461 98.27130364]


Threshold: 0.9
Validation Accuracy: 97.5467
Validation Precision Scores: [68.29268293 98.51166533]
Validation Recall Scores: [60.21505376 98.94949495]
Validation F1 Scores: [64.         98.73009474]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 512
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 26.6003, Val Loss: 22.6351
Epoch 10/50, Train Loss: 22.5553, Val Loss: 21.9814
Epoch 15/50, Train Loss: 18.9885, Val Loss: 20.3403
Epoch 20/50, Train Loss: 18.9350, Val Loss: 22.9871
Early stopping triggered after 23 epochs because no improvements over 10 subsequent epochs!
Epoch 23/50, Train Loss: 17.7580, Val Loss: 22.0324
The function took 44.00 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.4810
Training Precision Scores: [25.88084561 99.70145826]
Training Recall Scores: [93.95348837 88.24186992]
Training F1 Scores: [40.5826218 93.6222977]

Validation Accuracy: 88.7072
Validation Precision Scores: [23.01369863 99.59146618]
Validation Recall Scores: [90.32258065 88.64646465]
Validation F1 Scores: [36.68122271 93.80076956]

Threshold: 0.5
Validation Accuracy: 88.7072
Validation Precision Scores: [23.01369863 99.59146618]
Validation Recall Scores: [90.32258065 88.64646465]
Validation F1 Scores: [36.68122271 93.80076956]


Threshold: 0.6
Validation Accuracy: 92.3676
Validation Precision Scores: [30.41825095 99.43600868]
Validation Recall Scores: [86.02150538 92.60606061]
Validation F1 Scores: [44.94382022 95.89958159]


Threshold: 0.7
Validation Accuracy: 94.7430
Validation Precision Scores: [39.17525773 99.28390901]
Validation Recall Scores: [81.72043011 95.23232323]
Validation F1 Scores: [52.96167247 97.21592081]


Threshold: 0.8
Validation Accuracy: 96.7290
Validation Precision Scores: [53.4351145  99.05621666]
Validation Recall Scores: [75.2688172  97.53535354]
Validation F1 Scores: [62.5        98.28990228]


Threshold: 0.9
Validation Accuracy: 97.1184
Validation Precision Scores: [60.        98.5442782]
Validation Recall Scores: [61.29032258 98.46464646]
Validation F1 Scores: [60.63829787 98.50444624]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 64
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 26.9364, Val Loss: 22.6612
Epoch 10/50, Train Loss: 24.2907, Val Loss: 23.6795
Epoch 15/50, Train Loss: 19.9746, Val Loss: 21.4657
Epoch 20/50, Train Loss: 19.2911, Val Loss: 22.5909
Epoch 25/50, Train Loss: 19.9148, Val Loss: 22.2925
Early stopping triggered after 29 epochs because no improvements over 10 subsequent epochs!
Epoch 29/50, Train Loss: 17.3456, Val Loss: 20.8786
The function took 44.72 seconds to execute.
No description has been provided for this image
Training Accuracy: 87.8676
Training Precision Scores: [24.96932515 99.7337963 ]
Training Recall Scores: [94.65116279 87.57113821]
Training F1 Scores: [39.51456311 93.25757576]

Validation Accuracy: 87.1885
Validation Precision Scores: [20.64676617 99.53831948]
Validation Recall Scores: [89.24731183 87.11111111]
Validation F1 Scores: [33.53535354 92.91101056]

Threshold: 0.5
Validation Accuracy: 87.1885
Validation Precision Scores: [20.64676617 99.53831948]
Validation Recall Scores: [89.24731183 87.11111111]
Validation F1 Scores: [33.53535354 92.91101056]


Threshold: 0.6
Validation Accuracy: 93.0685
Validation Precision Scores: [32.06751055 99.27069927]
Validation Recall Scores: [81.72043011 93.49494949]
Validation F1 Scores: [46.06060606 96.2962963 ]


Threshold: 0.7
Validation Accuracy: 95.5218
Validation Precision Scores: [43.03797468 98.9626556 ]
Validation Recall Scores: [73.11827957 96.36363636]
Validation F1 Scores: [54.18326693 97.64585466]


Threshold: 0.8
Validation Accuracy: 96.7679
Validation Precision Scores: [54.09836066 98.89615699]
Validation Recall Scores: [70.96774194 97.73737374]
Validation F1 Scores: [61.39534884 98.31335094]


Threshold: 0.9
Validation Accuracy: 96.8458
Validation Precision Scores: [56.97674419 98.2272361 ]
Validation Recall Scores: [52.68817204 98.50505051]
Validation F1 Scores: [54.74860335 98.36594715]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 128
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 25.9277, Val Loss: 24.7019
Epoch 10/50, Train Loss: 20.6754, Val Loss: 21.6448
Epoch 15/50, Train Loss: 19.1554, Val Loss: 22.5969
Early stopping triggered after 19 epochs because no improvements over 10 subsequent epochs!
Epoch 19/50, Train Loss: 18.3732, Val Loss: 29.2883
The function took 29.92 seconds to execute.
No description has been provided for this image
Training Accuracy: 91.3437
Training Precision Scores: [31.35662063 99.51322049]
Training Recall Scores: [89.76744186 91.41260163]
Training F1 Scores: [46.47802529 95.29106415]

Validation Accuracy: 91.2773
Validation Precision Scores: [27.64505119 99.47252747]
Validation Recall Scores: [87.09677419 91.43434343]
Validation F1 Scores: [41.96891192 95.28421053]

Threshold: 0.5
Validation Accuracy: 91.2773
Validation Precision Scores: [27.64505119 99.47252747]
Validation Recall Scores: [87.09677419 91.43434343]
Validation F1 Scores: [41.96891192 95.28421053]


Threshold: 0.6
Validation Accuracy: 93.6916
Validation Precision Scores: [34.24657534 99.23371648]
Validation Recall Scores: [80.64516129 94.18181818]
Validation F1 Scores: [48.07692308 96.64179104]


Threshold: 0.7
Validation Accuracy: 95.5997
Validation Precision Scores: [43.90243902 99.12645591]
Validation Recall Scores: [77.41935484 96.28282828]
Validation F1 Scores: [56.0311284  97.68395163]


Threshold: 0.8
Validation Accuracy: 96.0670
Validation Precision Scores: [46.77419355 98.56792144]
Validation Recall Scores: [62.3655914  97.33333333]
Validation F1 Scores: [53.4562212  97.94673714]


Threshold: 0.9
Validation Accuracy: 97.0405
Validation Precision Scores: [60.         98.30849778]
Validation Recall Scores: [54.83870968 98.62626263]
Validation F1 Scores: [57.30337079 98.46712384]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 256
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 24.8474, Val Loss: 27.8234
Epoch 10/50, Train Loss: 21.2621, Val Loss: 25.1837
Epoch 15/50, Train Loss: 17.5756, Val Loss: 28.5194
Epoch 20/50, Train Loss: 16.9798, Val Loss: 19.8292
Epoch 25/50, Train Loss: 15.2126, Val Loss: 31.7794
Early stopping triggered after 30 epochs because no improvements over 10 subsequent epochs!
Epoch 30/50, Train Loss: 13.9144, Val Loss: 36.0606
The function took 51.71 seconds to execute.
No description has been provided for this image
Training Accuracy: 90.6621
Training Precision Scores: [30.59427733 99.85404738]
Training Recall Scores: [96.97674419 90.38617886]
Training F1 Scores: [46.51422197 94.88451486]

Validation Accuracy: 89.9922
Validation Precision Scores: [24.69135802 99.42067736]
Validation Recall Scores: [86.02150538 90.14141414]
Validation F1 Scores: [38.36930456 94.55393092]

Threshold: 0.5
Validation Accuracy: 89.9922
Validation Precision Scores: [24.69135802 99.42067736]
Validation Recall Scores: [86.02150538 90.14141414]
Validation F1 Scores: [38.36930456 94.55393092]


Threshold: 0.6
Validation Accuracy: 92.9128
Validation Precision Scores: [31.6872428  99.31182796]
Validation Recall Scores: [82.79569892 93.29292929]
Validation F1 Scores: [45.83333333 96.20833333]


Threshold: 0.7
Validation Accuracy: 94.9766
Validation Precision Scores: [39.88764045 99.07949791]
Validation Recall Scores: [76.34408602 95.67676768]
Validation F1 Scores: [52.39852399 97.34840699]


Threshold: 0.8
Validation Accuracy: 96.4953
Validation Precision Scores: [51.11111111 99.0135635 ]
Validation Recall Scores: [74.19354839 97.33333333]
Validation F1 Scores: [60.52631579 98.16625917]


Threshold: 0.9
Validation Accuracy: 97.5857
Validation Precision Scores: [67.41573034 98.66881807]
Validation Recall Scores: [64.51612903 98.82828283]
Validation F1 Scores: [65.93406593 98.74848607]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 320
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 34.2836, Val Loss: 33.5547
Epoch 10/50, Train Loss: 29.3603, Val Loss: 30.9909
Epoch 15/50, Train Loss: 26.5914, Val Loss: 39.7731
Epoch 20/50, Train Loss: 27.5414, Val Loss: 30.8492
Early stopping triggered after 24 epochs because no improvements over 10 subsequent epochs!
Epoch 24/50, Train Loss: 25.0730, Val Loss: 29.0656
The function took 44.87 seconds to execute.
No description has been provided for this image
Training Accuracy: 81.1100
Training Precision Scores: [17.28769497 99.61065059]
Training Recall Scores: [92.79069767 80.5995935 ]
Training F1 Scores: [29.14536158 89.10234805]

Validation Accuracy: 81.3863
Validation Precision Scores: [14.67889908 99.35739001]
Validation Recall Scores: [86.02150538 81.21212121]
Validation F1 Scores: [25.07836991 89.37305469]

Threshold: 0.5
Validation Accuracy: 81.3863
Validation Precision Scores: [14.67889908 99.35739001]
Validation Recall Scores: [86.02150538 81.21212121]
Validation F1 Scores: [25.07836991 89.37305469]


Threshold: 0.6
Validation Accuracy: 85.5140
Validation Precision Scores: [17.63341067 99.20449228]
Validation Recall Scores: [81.72043011 85.65656566]
Validation F1 Scores: [29.00763359 91.934085  ]


Threshold: 0.7
Validation Accuracy: 88.6682
Validation Precision Scores: [20.35928144 98.88093107]
Validation Recall Scores: [73.11827957 89.25252525]
Validation F1 Scores: [31.8501171  93.82034402]


Threshold: 0.8
Validation Accuracy: 91.5888
Validation Precision Scores: [22.66666667 98.20742638]
Validation Recall Scores: [54.83870968 92.96969697]
Validation F1 Scores: [32.0754717  95.51681196]


Threshold: 0.9
Validation Accuracy: 95.7944
Validation Precision Scores: [34.69387755 96.98292973]
Validation Recall Scores: [18.27956989 98.70707071]
Validation F1 Scores: [23.94366197 97.83740489]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 384
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 34.6684, Val Loss: 33.8397
Epoch 10/50, Train Loss: 29.1103, Val Loss: 33.4789
Epoch 15/50, Train Loss: 26.1147, Val Loss: 33.0259
Epoch 20/50, Train Loss: 25.7866, Val Loss: 31.6822
Epoch 25/50, Train Loss: 27.2418, Val Loss: 32.1745
Early stopping triggered after 27 epochs because no improvements over 10 subsequent epochs!
Epoch 27/50, Train Loss: 23.5823, Val Loss: 34.5373
The function took 52.13 seconds to execute.
No description has been provided for this image
Training Accuracy: 78.8608
Training Precision Scores: [15.95619867 99.71476728]
Training Recall Scores: [94.88372093 78.16056911]
Training F1 Scores: [27.31837965 87.63174386]

Validation Accuracy: 78.1153
Validation Precision Scores: [13.07086614 99.48266943]
Validation Recall Scores: [89.24731183 77.6969697 ]
Validation F1 Scores: [22.8021978  87.25045372]

Threshold: 0.5
Validation Accuracy: 78.1153
Validation Precision Scores: [13.07086614 99.48266943]
Validation Recall Scores: [89.24731183 77.6969697 ]
Validation F1 Scores: [22.8021978  87.25045372]


Threshold: 0.6
Validation Accuracy: 82.6713
Validation Precision Scores: [15.625      99.36770428]
Validation Recall Scores: [86.02150538 82.54545455]
Validation F1 Scores: [26.44628099 90.17876848]


Threshold: 0.7
Validation Accuracy: 86.4097
Validation Precision Scores: [18.15920398 99.07663897]
Validation Recall Scores: [78.49462366 86.70707071]
Validation F1 Scores: [29.49494949 92.48006895]


Threshold: 0.8
Validation Accuracy: 91.3162
Validation Precision Scores: [22.22222222 98.24335904]
Validation Recall Scores: [55.91397849 92.64646465]
Validation F1 Scores: [31.80428135 95.3628613 ]


Threshold: 0.9
Validation Accuracy: 96.0670
Validation Precision Scores: [42.85714286 97.25318471]
Validation Recall Scores: [25.80645161 98.70707071]
Validation F1 Scores: [32.2147651  97.97473431]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 512
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 25.4428, Val Loss: 24.7843
Epoch 10/50, Train Loss: 21.3522, Val Loss: 25.0574
Epoch 15/50, Train Loss: 18.2956, Val Loss: 21.8457
Epoch 20/50, Train Loss: 15.8972, Val Loss: 27.3139
Early stopping triggered after 25 epochs because no improvements over 10 subsequent epochs!
Epoch 25/50, Train Loss: 18.2115, Val Loss: 22.7816
The function took 52.40 seconds to execute.
No description has been provided for this image
Training Accuracy: 91.1490
Training Precision Scores: [31.33281372 99.68843886]
Training Recall Scores: [93.48837209 91.04674797]
Training F1 Scores: [46.9352014  95.17182769]

Validation Accuracy: 91.1215
Validation Precision Scores: [26.96245734 99.38461538]
Validation Recall Scores: [84.94623656 91.35353535]
Validation F1 Scores: [40.93264249 95.2       ]

Threshold: 0.5
Validation Accuracy: 91.1215
Validation Precision Scores: [26.96245734 99.38461538]
Validation Recall Scores: [84.94623656 91.35353535]
Validation F1 Scores: [40.93264249 95.2       ]


Threshold: 0.6
Validation Accuracy: 93.3022
Validation Precision Scores: [32.44444444 99.14639351]
Validation Recall Scores: [78.49462366 93.85858586]
Validation F1 Scores: [45.91194969 96.43005396]


Threshold: 0.7
Validation Accuracy: 95.2492
Validation Precision Scores: [41.21212121 98.95963379]
Validation Recall Scores: [73.11827957 96.08080808]
Validation F1 Scores: [52.71317829 97.49897499]


Threshold: 0.8
Validation Accuracy: 96.4953
Validation Precision Scores: [51.23966942 98.73314262]
Validation Recall Scores: [66.66666667 97.61616162]
Validation F1 Scores: [57.94392523 98.17147501]


Threshold: 0.9
Validation Accuracy: 97.0405
Validation Precision Scores: [61.33333333 98.11472122]
Validation Recall Scores: [49.46236559 98.82828283]
Validation F1 Scores: [54.76190476 98.47020934]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 128
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 26.1242, Val Loss: 21.9478
Epoch 10/50, Train Loss: 23.0291, Val Loss: 22.9733
Epoch 15/50, Train Loss: 18.3506, Val Loss: 26.1328
Epoch 20/50, Train Loss: 18.0447, Val Loss: 21.2710
Early stopping triggered after 24 epochs because no improvements over 10 subsequent epochs!
Epoch 24/50, Train Loss: 16.1969, Val Loss: 23.2196
The function took 45.32 seconds to execute.
No description has been provided for this image
Training Accuracy: 90.1461
Training Precision Scores: [28.79008746 99.60665318]
Training Recall Scores: [91.86046512 90.07113821]
Training F1 Scores: [43.84017758 94.59921016]

Validation Accuracy: 89.6028
Validation Precision Scores: [24.41176471 99.55116697]
Validation Recall Scores: [89.24731183 89.61616162]
Validation F1 Scores: [38.33718245 94.3227727 ]

Threshold: 0.5
Validation Accuracy: 89.6028
Validation Precision Scores: [24.41176471 99.55116697]
Validation Recall Scores: [89.24731183 89.61616162]
Validation F1 Scores: [38.33718245 94.3227727 ]


Threshold: 0.6
Validation Accuracy: 92.0561
Validation Precision Scores: [28.73563218 99.21976593]
Validation Recall Scores: [80.64516129 92.48484848]
Validation F1 Scores: [42.37288136 95.73400251]


Threshold: 0.7
Validation Accuracy: 94.1589
Validation Precision Scores: [35.23316062 98.94736842]
Validation Recall Scores: [73.11827957 94.94949495]
Validation F1 Scores: [47.55244755 96.90721649]


Threshold: 0.8
Validation Accuracy: 95.6776
Validation Precision Scores: [44.         98.88337469]
Validation Recall Scores: [70.96774194 96.60606061]
Validation F1 Scores: [54.32098765 97.7314531 ]


Threshold: 0.9
Validation Accuracy: 96.9237
Validation Precision Scores: [57.95454545 98.30645161]
Validation Recall Scores: [54.83870968 98.50505051]
Validation F1 Scores: [56.35359116 98.40565086]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 256
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 25.7412, Val Loss: 27.8013
Epoch 10/50, Train Loss: 22.1580, Val Loss: 23.7855
Epoch 15/50, Train Loss: 20.9876, Val Loss: 29.3248
Early stopping triggered after 20 epochs because no improvements over 10 subsequent epochs!
Epoch 20/50, Train Loss: 20.3855, Val Loss: 27.4963
The function took 41.81 seconds to execute.
No description has been provided for this image
Training Accuracy: 91.0808
Training Precision Scores: [30.59105431 99.47882014]
Training Recall Scores: [89.06976744 91.16869919]
Training F1 Scores: [45.54102259 95.14264503]

Validation Accuracy: 90.3816
Validation Precision Scores: [24.67105263 99.204947  ]
Validation Recall Scores: [80.64516129 90.74747475]
Validation F1 Scores: [37.78337531 94.78792994]

Threshold: 0.5
Validation Accuracy: 90.3816
Validation Precision Scores: [24.67105263 99.204947  ]
Validation Recall Scores: [80.64516129 90.74747475]
Validation F1 Scores: [37.78337531 94.78792994]


Threshold: 0.6
Validation Accuracy: 94.1199
Validation Precision Scores: [35.5        99.07094595]
Validation Recall Scores: [76.34408602 94.78787879]
Validation F1 Scores: [48.46416382 96.88209787]


Threshold: 0.7
Validation Accuracy: 95.3271
Validation Precision Scores: [41.71779141 98.96049896]
Validation Recall Scores: [73.11827957 96.16161616]
Validation F1 Scores: [53.125      97.54098361]


Threshold: 0.8
Validation Accuracy: 96.3396
Validation Precision Scores: [49.51456311 98.29614604]
Validation Recall Scores: [54.83870968 97.8989899 ]
Validation F1 Scores: [52.04081633 98.09716599]


Threshold: 0.9
Validation Accuracy: 96.9237
Validation Precision Scores: [59.21052632 98.07383628]
Validation Recall Scores: [48.38709677 98.74747475]
Validation F1 Scores: [53.25443787 98.40950272]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 320
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 27.5092, Val Loss: 28.0371
Epoch 10/50, Train Loss: 22.6324, Val Loss: 31.6975
Early stopping triggered after 15 epochs because no improvements over 10 subsequent epochs!
Epoch 15/50, Train Loss: 20.4260, Val Loss: 33.3761
The function took 32.45 seconds to execute.
No description has been provided for this image
Training Accuracy: 89.3768
Training Precision Scores: [24.90508732 98.86071708]
Training Recall Scores: [76.27906977 89.94918699]
Training F1 Scores: [37.55008586 94.19464694]

Validation Accuracy: 88.9798
Validation Precision Scores: [21.21212121 98.97229669]
Validation Recall Scores: [75.2688172  89.49494949]
Validation F1 Scores: [33.09692671 93.99533206]

Threshold: 0.5
Validation Accuracy: 88.9798
Validation Precision Scores: [21.21212121 98.97229669]
Validation Recall Scores: [75.2688172  89.49494949]
Validation F1 Scores: [33.09692671 93.99533206]


Threshold: 0.6
Validation Accuracy: 93.3022
Validation Precision Scores: [30.15075377 98.60700718]
Validation Recall Scores: [64.51612903 94.38383838]
Validation F1 Scores: [41.09589041 96.44921552]


Threshold: 0.7
Validation Accuracy: 94.9766
Validation Precision Scores: [36.36363636 98.15270936]
Validation Recall Scores: [51.61290323 96.60606061]
Validation F1 Scores: [42.66666667 97.37324374]


Threshold: 0.8
Validation Accuracy: 95.8333
Validation Precision Scores: [42.39130435 97.819063  ]
Validation Recall Scores: [41.93548387 97.85858586]
Validation F1 Scores: [42.16216216 97.83882044]


Threshold: 0.9
Validation Accuracy: 96.4564
Validation Precision Scores: [51.51515152 97.64188649]
Validation Recall Scores: [36.55913978 98.70707071]
Validation F1 Scores: [42.7672956  98.17158931]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 384
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 25.8026, Val Loss: 22.9581
Epoch 10/50, Train Loss: 23.5490, Val Loss: 23.2874
Epoch 15/50, Train Loss: 21.9899, Val Loss: 24.1232
Epoch 20/50, Train Loss: 19.4286, Val Loss: 26.8190
Early stopping triggered after 24 epochs because no improvements over 10 subsequent epochs!
Epoch 24/50, Train Loss: 17.0860, Val Loss: 23.2527
The function took 52.89 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.8510
Training Precision Scores: [26.40264026 99.65733866]
Training Recall Scores: [93.02325581 88.66869919]
Training F1 Scores: [41.1311054  93.84243076]

Validation Accuracy: 88.3956
Validation Precision Scores: [22.66666667 99.63520292]
Validation Recall Scores: [91.39784946 88.28282828]
Validation F1 Scores: [36.32478632 93.61610968]

Threshold: 0.5
Validation Accuracy: 88.3956
Validation Precision Scores: [22.66666667 99.63520292]
Validation Recall Scores: [91.39784946 88.28282828]
Validation F1 Scores: [36.32478632 93.61610968]


Threshold: 0.6
Validation Accuracy: 92.7570
Validation Precision Scores: [31.3253012  99.35316947]
Validation Recall Scores: [83.87096774 93.09090909]
Validation F1 Scores: [45.61403509 96.12015019]


Threshold: 0.7
Validation Accuracy: 96.1838
Validation Precision Scores: [48.27586207 99.05076352]
Validation Recall Scores: [75.2688172  96.96969697]
Validation F1 Scores: [58.82352941 97.99918334]


Threshold: 0.8
Validation Accuracy: 96.8069
Validation Precision Scores: [55.23809524 98.57896874]
Validation Recall Scores: [62.3655914 98.1010101]
Validation F1 Scores: [58.58585859 98.33940867]


Threshold: 0.9
Validation Accuracy: 97.3910
Validation Precision Scores: [67.10526316 98.31460674]
Validation Recall Scores: [54.83870968 98.98989899]
Validation F1 Scores: [60.35502959 98.65109724]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 512
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 28.0761, Val Loss: 23.0171
Epoch 10/50, Train Loss: 24.0684, Val Loss: 22.7654
Early stopping triggered after 13 epochs because no improvements over 10 subsequent epochs!
Epoch 13/50, Train Loss: 21.2287, Val Loss: 23.5803
The function took 30.52 seconds to execute.
No description has been provided for this image
Training Accuracy: 90.0292
Training Precision Scores: [25.85365854 98.76106195]
Training Recall Scores: [73.95348837 90.73170732]
Training F1 Scores: [38.31325301 94.57627119]

Validation Accuracy: 90.3816
Validation Precision Scores: [24.50331126 99.16151809]
Validation Recall Scores: [79.56989247 90.78787879]
Validation F1 Scores: [37.46835443 94.79012866]

Threshold: 0.5
Validation Accuracy: 90.3816
Validation Precision Scores: [24.50331126 99.16151809]
Validation Recall Scores: [79.56989247 90.78787879]
Validation F1 Scores: [37.46835443 94.79012866]


Threshold: 0.6
Validation Accuracy: 93.4190
Validation Precision Scores: [32.56880734 99.06382979]
Validation Recall Scores: [76.34408602 94.06060606]
Validation F1 Scores: [45.65916399 96.49740933]


Threshold: 0.7
Validation Accuracy: 95.5218
Validation Precision Scores: [42.66666667 98.8006617 ]
Validation Recall Scores: [68.8172043  96.52525253]
Validation F1 Scores: [52.67489712 97.64970366]


Threshold: 0.8
Validation Accuracy: 96.8847
Validation Precision Scores: [56.84210526 98.42296805]
Validation Recall Scores: [58.06451613 98.34343434]
Validation F1 Scores: [57.44680851 98.38318513]


Threshold: 0.9
Validation Accuracy: 97.4299
Validation Precision Scores: [69.01408451 98.23788546]
Validation Recall Scores: [52.68817204 99.11111111]
Validation F1 Scores: [59.75609756 98.67256637]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 256
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 24.8792, Val Loss: 23.6952
Epoch 10/50, Train Loss: 20.0086, Val Loss: 22.8912
Epoch 15/50, Train Loss: 21.8312, Val Loss: 30.5929
Early stopping triggered after 20 epochs because no improvements over 10 subsequent epochs!
Epoch 20/50, Train Loss: 18.4261, Val Loss: 25.4104
The function took 50.86 seconds to execute.
No description has been provided for this image
Training Accuracy: 89.8832
Training Precision Scores: [28.29650748 99.62783354]
Training Recall Scores: [92.3255814  89.77642276]
Training F1 Scores: [43.31696672 94.44592933]

Validation Accuracy: 88.9798
Validation Precision Scores: [22.70114943 99.36936937]
Validation Recall Scores: [84.94623656 89.13131313]
Validation F1 Scores: [35.8276644  93.97231097]

Threshold: 0.5
Validation Accuracy: 88.9798
Validation Precision Scores: [22.70114943 99.36936937]
Validation Recall Scores: [84.94623656 89.13131313]
Validation F1 Scores: [35.8276644  93.97231097]


Threshold: 0.6
Validation Accuracy: 92.4455
Validation Precision Scores: [29.7188755  99.18068133]
Validation Recall Scores: [79.56989247 92.92929293]
Validation F1 Scores: [43.2748538  95.95327493]


Threshold: 0.7
Validation Accuracy: 95.6386
Validation Precision Scores: [44.02515723 99.04524699]
Validation Recall Scores: [75.2688172 96.4040404]
Validation F1 Scores: [55.55555556 97.70679771]


Threshold: 0.8
Validation Accuracy: 96.6900
Validation Precision Scores: [53.17460317 98.93529894]
Validation Recall Scores: [72.04301075 97.61616162]
Validation F1 Scores: [61.18721461 98.27130364]


Threshold: 0.9
Validation Accuracy: 97.5857
Validation Precision Scores: [67.81609195 98.62958484]
Validation Recall Scores: [63.44086022 98.86868687]
Validation F1 Scores: [65.55555556 98.74899112]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 320
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 25.9213, Val Loss: 30.6943
Epoch 10/50, Train Loss: 23.8911, Val Loss: 27.2544
Early stopping triggered after 13 epochs because no improvements over 10 subsequent epochs!
Epoch 13/50, Train Loss: 18.2578, Val Loss: 25.8883
The function took 34.61 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.8257
Training Precision Scores: [21.97935641 99.21141134]
Training Recall Scores: [84.18604651 86.94105691]
Training F1 Scores: [34.85796822 92.67183015]

Validation Accuracy: 86.2150
Validation Precision Scores: [18.70503597 99.30264993]
Validation Recall Scores: [83.87096774 86.3030303 ]
Validation F1 Scores: [30.58823529 92.34760052]

Threshold: 0.5
Validation Accuracy: 86.2150
Validation Precision Scores: [18.70503597 99.30264993]
Validation Recall Scores: [83.87096774 86.3030303 ]
Validation F1 Scores: [30.58823529 92.34760052]


Threshold: 0.6
Validation Accuracy: 92.1340
Validation Precision Scores: [27.75510204 98.92380542]
Validation Recall Scores: [73.11827957 92.84848485]
Validation F1 Scores: [40.23668639 95.78991246]


Threshold: 0.7
Validation Accuracy: 95.0935
Validation Precision Scores: [39.35483871 98.67384998]
Validation Recall Scores: [65.59139785 96.2020202 ]
Validation F1 Scores: [49.19354839 97.42225859]


Threshold: 0.8
Validation Accuracy: 96.4174
Validation Precision Scores: [50.60240964 97.94768612]
Validation Recall Scores: [45.16129032 98.34343434]
Validation F1 Scores: [47.72727273 98.14516129]


Threshold: 0.9
Validation Accuracy: 96.9237
Validation Precision Scores: [63.46153846 97.61526232]
Validation Recall Scores: [35.48387097 99.23232323]
Validation F1 Scores: [45.51724138 98.41715087]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 384
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 27.2971, Val Loss: 28.6508
Epoch 10/50, Train Loss: 26.5712, Val Loss: 25.4273
Epoch 15/50, Train Loss: 20.6089, Val Loss: 23.0051
Early stopping triggered after 16 epochs because no improvements over 10 subsequent epochs!
Epoch 16/50, Train Loss: 19.0696, Val Loss: 30.0144
The function took 45.21 seconds to execute.
No description has been provided for this image
Training Accuracy: 90.8666
Training Precision Scores: [29.4165316  99.25852147]
Training Recall Scores: [84.41860465 91.14837398]
Training F1 Scores: [43.62980769 95.03072685]

Validation Accuracy: 91.2383
Validation Precision Scores: [26.76056338 99.25569177]
Validation Recall Scores: [81.72043011 91.5959596 ]
Validation F1 Scores: [40.31830239 95.27211599]

Threshold: 0.5
Validation Accuracy: 91.2383
Validation Precision Scores: [26.76056338 99.25569177]
Validation Recall Scores: [81.72043011 91.5959596 ]
Validation F1 Scores: [40.31830239 95.27211599]


Threshold: 0.6
Validation Accuracy: 93.4579
Validation Precision Scores: [33.03167421 99.14784832]
Validation Recall Scores: [78.49462366 94.02020202]
Validation F1 Scores: [46.49681529 96.51596848]


Threshold: 0.7
Validation Accuracy: 94.9766
Validation Precision Scores: [39.88764045 99.07949791]
Validation Recall Scores: [76.34408602 95.67676768]
Validation F1 Scores: [52.39852399 97.34840699]


Threshold: 0.8
Validation Accuracy: 96.1449
Validation Precision Scores: [47.65625    98.68852459]
Validation Recall Scores: [65.59139785 97.29292929]
Validation F1 Scores: [55.20361991 97.98575788]


Threshold: 0.9
Validation Accuracy: 96.8458
Validation Precision Scores: [56.38297872 98.38318513]
Validation Recall Scores: [56.98924731 98.34343434]
Validation F1 Scores: [56.68449198 98.36330572]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 512
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 25.2597, Val Loss: 26.4426
Epoch 10/50, Train Loss: 22.2792, Val Loss: 28.7916
Epoch 15/50, Train Loss: 22.3347, Val Loss: 27.5345
Epoch 20/50, Train Loss: 19.1259, Val Loss: 36.1332
Early stopping triggered after 21 epochs because no improvements over 10 subsequent epochs!
Epoch 21/50, Train Loss: 18.6059, Val Loss: 37.0555
The function took 62.19 seconds to execute.
No description has been provided for this image
Training Accuracy: 91.0613
Training Precision Scores: [30.         99.29281768]
Training Recall Scores: [85.11627907 91.32113821]
Training F1 Scores: [44.36363636 95.14028587]

Validation Accuracy: 90.0701
Validation Precision Scores: [24.03846154 99.20212766]
Validation Recall Scores: [80.64516129 90.42424242]
Validation F1 Scores: [37.03703704 94.61001902]

Threshold: 0.5
Validation Accuracy: 90.0701
Validation Precision Scores: [24.03846154 99.20212766]
Validation Recall Scores: [80.64516129 90.42424242]
Validation F1 Scores: [37.03703704 94.61001902]


Threshold: 0.6
Validation Accuracy: 93.4969
Validation Precision Scores: [32.21153846 98.89830508]
Validation Recall Scores: [72.04301075 94.3030303 ]
Validation F1 Scores: [44.51827243 96.54601861]


Threshold: 0.7
Validation Accuracy: 95.6386
Validation Precision Scores: [42.63565891 98.44198442]
Validation Recall Scores: [59.13978495 97.01010101]
Validation F1 Scores: [49.54954955 97.72079772]


Threshold: 0.8
Validation Accuracy: 96.4953
Validation Precision Scores: [51.68539326 98.10407422]
Validation Recall Scores: [49.46236559 98.26262626]
Validation F1 Scores: [50.54945055 98.18328623]


Threshold: 0.9
Validation Accuracy: 97.0405
Validation Precision Scores: [62.31884058 97.99919968]
Validation Recall Scores: [46.23655914 98.94949495]
Validation F1 Scores: [53.08641975 98.47205468]


In [27]:
# 16 and hidden dim 256 threshold 0.9

embedding_size=16
hidden_dim=256
threshold = 0.9 

# Create embedding layer with the given vocabulary size and embedding size
vocabulary_size = len(vectorizer.vocabulary_) + 1  # Add 1 for out-of-vocabulary
embeddings = nn.Embedding(vocabulary_size, embedding_size)
X_train_graph = GraphDataset(X_train, y_train, embeddings)
X_val_graph = GraphDataset(X_val, y_val, embeddings)
X_test_graph = GraphDataset(X_test, y_test, embeddings)

# Create DataLoader for batching
# NOTICE: This data loader is from torch_geometric.loader!! It is not from torch.utils.data
batch_size = 64
train_loader = torch_geometric.loader.DataLoader(X_train_graph, batch_size=batch_size, shuffle=True)
val_loader = torch_geometric.loader.DataLoader(X_val_graph, batch_size=batch_size, shuffle=False)
test_loader = torch_geometric.loader.DataLoader(X_test_graph, batch_size=batch_size, shuffle=False)

class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train_original), y=y_train_original)  # Computing class weights
class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
num_epochs = 50
model = GraphSAGE(input_dim=embedding_size, hidden_dim=hidden_dim, output_dim=2).to(device)
optimizer = optim.AdamW(model.parameters(), lr=0.01, weight_decay=0.01)
training_loop(model, train_loader, val_loader, X_train_graph, X_val_graph, device, optimizer, num_epochs, criterion)

test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model_with_threshold(model,val_loader,device,threshold=threshold)

stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 29.8540, Val Loss: 32.2394
Epoch 10/50, Train Loss: 23.8979, Val Loss: 23.4067
Epoch 15/50, Train Loss: 22.0149, Val Loss: 25.1712
Early stopping triggered after 20 epochs because no improvements over 10 subsequent epochs!
Epoch 20/50, Train Loss: 20.2248, Val Loss: 23.6084
The function took 29.33 seconds to execute.
No description has been provided for this image
Test Accuracy: 96.5732
Test Precision Scores: [53.01204819 98.02816901]
Test Recall Scores: [47.31182796 98.42424242]
Test F1 Scores: [50.         98.22580645]

Out[27]:
<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7a93abb7ff10>
No description has been provided for this image
No description has been provided for this image

GAT¶

In [17]:
embedding_sizes = [8,16,32,64,128,256]
hidden_dims = [8,16,32,64,128,256,320, 384, 512]

for i in range(len(embedding_sizes)):
    for j in range(i,len(hidden_dims)):
        embedding_size=embedding_sizes[i]
        hidden_dim=hidden_dims[j]
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------")
        print(f"Trying with embedding size {embedding_size} and hidden dim {hidden_dim}")

        # Create embedding layer with the given vocabulary size and embedding size
        vocabulary_size = len(vectorizer.vocabulary_) + 1  # Add 1 for out-of-vocabulary
        embeddings = nn.Embedding(vocabulary_size, embedding_size)
        X_train_graph = GraphDataset(X_train, y_train, embeddings)
        X_val_graph = GraphDataset(X_val, y_val, embeddings)
        X_test_graph = GraphDataset(X_test, y_test, embeddings)
        
        # Create DataLoader for batching
        # NOTICE: This data loader is from torch_geometric.loader!! It is not from torch.utils.data
        batch_size = 64
        train_loader = torch_geometric.loader.DataLoader(X_train_graph, batch_size=batch_size, shuffle=True)
        val_loader = torch_geometric.loader.DataLoader(X_val_graph, batch_size=batch_size, shuffle=False)
        test_loader = torch_geometric.loader.DataLoader(X_test_graph, batch_size=batch_size, shuffle=False)
        
        class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train_original), y=y_train_original)  # Computing class weights
        class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
        criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
        num_epochs = 50
        model = GAT(input_dim=embedding_size, hidden_dim=hidden_dim, output_dim=2).to(device)
        optimizer = optim.AdamW(model.parameters(), lr=0.01, weight_decay=0.01)
        training_loop(model, train_loader, val_loader, X_train_graph, X_val_graph, device, optimizer, num_epochs, criterion)
        
        train_accuracy, train_precisionscores, train_recallscores, train_f1scores, _, _ = testing_model(model,train_loader,device)
        val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model(model,val_loader,device)
        #test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model(model,test_loader,device)
        
        stats(train_accuracy, train_precisionscores, train_recallscores, train_f1scores, 'Training')
        stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
        #stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')
        #ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
        #ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')
        
        # Test with different thresholds
        for threshold in [0.5, 0.6, 0.7, 0.8, 0.9]:
            print(f"Threshold: {threshold}")
            val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model_with_threshold(model,val_loader,device,threshold=threshold)
            stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
            print()
-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 8
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 39.0102, Val Loss: 38.1614
Epoch 10/50, Train Loss: 33.1951, Val Loss: 32.8341
Epoch 15/50, Train Loss: 30.4372, Val Loss: 30.8448
Epoch 20/50, Train Loss: 30.3905, Val Loss: 31.0182
Epoch 25/50, Train Loss: 28.0026, Val Loss: 26.8883
Epoch 30/50, Train Loss: 27.4088, Val Loss: 27.7825
Early stopping triggered after 33 epochs because no improvements over 10 subsequent epochs!
Epoch 33/50, Train Loss: 26.3089, Val Loss: 29.9954
The function took 56.98 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.1831
Training Precision Scores: [20.7223209  99.06770773]
Training Recall Scores: [81.39534884 86.39227642]
Training F1 Scores: [33.03445021 92.29683513]

Validation Accuracy: 86.7212
Validation Precision Scores: [19.45812808 99.35245143]
Validation Recall Scores: [84.94623656 86.78787879]
Validation F1 Scores: [31.66332665 92.6461074 ]

Threshold: 0.5
Validation Accuracy: 86.7212
Validation Precision Scores: [19.45812808 99.35245143]
Validation Recall Scores: [84.94623656 86.78787879]
Validation F1 Scores: [31.66332665 92.6461074 ]


Threshold: 0.6
Validation Accuracy: 90.9657
Validation Precision Scores: [25.26690391 99.0380411 ]
Validation Recall Scores: [76.34408602 91.51515152]
Validation F1 Scores: [37.96791444 95.12809744]


Threshold: 0.7
Validation Accuracy: 94.4315
Validation Precision Scores: [35.46511628 98.66444073]
Validation Recall Scores: [65.59139785 95.51515152]
Validation F1 Scores: [46.03773585 97.06425785]


Threshold: 0.8
Validation Accuracy: 96.3785
Validation Precision Scores: [50.         98.10177706]
Validation Recall Scores: [49.46236559 98.14141414]
Validation F1 Scores: [49.72972973 98.1215916 ]


Threshold: 0.9
Validation Accuracy: 97.0405
Validation Precision Scores: [68.08510638 97.58032527]
Validation Recall Scores: [34.40860215 99.39393939]
Validation F1 Scores: [45.71428571 98.47878303]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 16
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 38.8971, Val Loss: 35.2171
Epoch 10/50, Train Loss: 33.2826, Val Loss: 31.8862
Epoch 15/50, Train Loss: 31.6364, Val Loss: 27.9429
Epoch 20/50, Train Loss: 31.4714, Val Loss: 29.5137
Epoch 25/50, Train Loss: 28.5403, Val Loss: 24.5040
Epoch 30/50, Train Loss: 29.9110, Val Loss: 28.7493
Early stopping triggered after 35 epochs because no improvements over 10 subsequent epochs!
Epoch 35/50, Train Loss: 27.2175, Val Loss: 24.9689
The function took 60.12 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.0526
Training Precision Scores: [23.27297116 99.05456202]
Training Recall Scores: [80.69767442 88.37398374]
Training F1 Scores: [36.12701718 93.40995757]

Validation Accuracy: 89.0187
Validation Precision Scores: [22.12389381 99.19246299]
Validation Recall Scores: [80.64516129 89.33333333]
Validation F1 Scores: [34.72222222 94.00510204]

Threshold: 0.5
Validation Accuracy: 89.0187
Validation Precision Scores: [22.12389381 99.19246299]
Validation Recall Scores: [80.64516129 89.33333333]
Validation F1 Scores: [34.72222222 94.00510204]


Threshold: 0.6
Validation Accuracy: 92.2118
Validation Precision Scores: [28.51405622 99.05131522]
Validation Recall Scores: [76.34408602 92.80808081]
Validation F1 Scores: [41.52046784 95.82811848]


Threshold: 0.7
Validation Accuracy: 94.0421
Validation Precision Scores: [32.75862069 98.4962406 ]
Validation Recall Scores: [61.29032258 95.27272727]
Validation F1 Scores: [42.69662921 96.85767098]


Threshold: 0.8
Validation Accuracy: 96.1838
Validation Precision Scores: [47.36842105 98.05903761]
Validation Recall Scores: [48.38709677 97.97979798]
Validation F1 Scores: [47.87234043 98.01940178]


Threshold: 0.9
Validation Accuracy: 96.8069
Validation Precision Scores: [61.2244898  97.49900754]
Validation Recall Scores: [32.25806452 99.23232323]
Validation F1 Scores: [42.25352113 98.35802964]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 32
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 41.5580, Val Loss: 57.4460
Epoch 10/50, Train Loss: 35.8319, Val Loss: 33.9315
Epoch 15/50, Train Loss: 33.2896, Val Loss: 34.8361
Epoch 20/50, Train Loss: 31.0224, Val Loss: 40.6379
Epoch 25/50, Train Loss: 28.6341, Val Loss: 31.0588
Epoch 30/50, Train Loss: 28.1938, Val Loss: 29.2019
Epoch 35/50, Train Loss: 25.9330, Val Loss: 27.9621
Epoch 40/50, Train Loss: 26.5195, Val Loss: 28.8038
Epoch 45/50, Train Loss: 25.4159, Val Loss: 26.9866
Epoch 50/50, Train Loss: 25.6434, Val Loss: 30.0298
The function took 86.10 seconds to execute.
No description has been provided for this image
Training Accuracy: 85.7936
Training Precision Scores: [20.51575931 99.15542522]
Training Recall Scores: [83.25581395 85.90447154]
Training F1 Scores: [32.91954023 92.05554043]

Validation Accuracy: 85.6308
Validation Precision Scores: [18.05555556 99.29775281]
Validation Recall Scores: [83.87096774 85.6969697 ]
Validation F1 Scores: [29.71428571 91.99739753]

Threshold: 0.5
Validation Accuracy: 85.6308
Validation Precision Scores: [18.05555556 99.29775281]
Validation Recall Scores: [83.87096774 85.6969697 ]
Validation F1 Scores: [29.71428571 91.99739753]


Threshold: 0.6
Validation Accuracy: 90.9657
Validation Precision Scores: [25.78397213 99.167032  ]
Validation Recall Scores: [79.56989247 91.39393939]
Validation F1 Scores: [38.94736842 95.12195122]


Threshold: 0.7
Validation Accuracy: 93.4190
Validation Precision Scores: [31.55339806 98.81456393]
Validation Recall Scores: [69.89247312 94.3030303 ]
Validation F1 Scores: [43.47826087 96.50609882]


Threshold: 0.8
Validation Accuracy: 95.5997
Validation Precision Scores: [41.93548387 98.32242226]
Validation Recall Scores: [55.91397849 97.09090909]
Validation F1 Scores: [47.92626728 97.70278512]


Threshold: 0.9
Validation Accuracy: 96.7679
Validation Precision Scores: [58.33333333 97.68740032]
Validation Recall Scores: [37.6344086  98.98989899]
Validation F1 Scores: [45.75163399 98.33433674]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 64
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 44.9863, Val Loss: 43.3956
Epoch 10/50, Train Loss: 43.3156, Val Loss: 35.1859
Epoch 15/50, Train Loss: 38.0289, Val Loss: 37.2140
Epoch 20/50, Train Loss: 33.1039, Val Loss: 30.1235
Epoch 25/50, Train Loss: 32.1548, Val Loss: 29.6338
Epoch 30/50, Train Loss: 30.4889, Val Loss: 32.2712
Early stopping triggered after 33 epochs because no improvements over 10 subsequent epochs!
Epoch 33/50, Train Loss: 29.4609, Val Loss: 29.1637
The function took 59.00 seconds to execute.
No description has been provided for this image
Training Accuracy: 79.3866
Training Precision Scores: [14.95637723 99.10975455]
Training Recall Scores: [83.72093023 79.19715447]
Training F1 Scores: [25.3789214  88.04157487]

Validation Accuracy: 78.8941
Validation Precision Scores: [12.64559068 99.13573971]
Validation Recall Scores: [81.72043011 78.78787879]
Validation F1 Scores: [21.90201729 87.79828906]

Threshold: 0.5
Validation Accuracy: 78.8941
Validation Precision Scores: [12.64559068 99.13573971]
Validation Recall Scores: [81.72043011 78.78787879]
Validation F1 Scores: [21.90201729 87.79828906]


Threshold: 0.6
Validation Accuracy: 86.0981
Validation Precision Scores: [17.48768473 98.98242368]
Validation Recall Scores: [76.34408602 86.46464646]
Validation F1 Scores: [28.45691383 92.30105672]


Threshold: 0.7
Validation Accuracy: 91.7056
Validation Precision Scores: [25.40983607 98.66609294]
Validation Recall Scores: [66.66666667 92.64646465]
Validation F1 Scores: [36.79525223 95.56157533]


Threshold: 0.8
Validation Accuracy: 94.5483
Validation Precision Scores: [35.03184713 98.4238905 ]
Validation Recall Scores: [59.13978495 95.87878788]
Validation F1 Scores: [44.         97.13467049]


Threshold: 0.9
Validation Accuracy: 96.5343
Validation Precision Scores: [52.7027027  97.83480353]
Validation Recall Scores: [41.93548387 98.58585859]
Validation F1 Scores: [46.70658683 98.20889515]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 128
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 46.6823, Val Loss: 41.9380
Epoch 10/50, Train Loss: 40.7365, Val Loss: 48.2736
Epoch 15/50, Train Loss: 37.8866, Val Loss: 36.3605
Epoch 20/50, Train Loss: 36.7250, Val Loss: 49.6080
Epoch 25/50, Train Loss: 35.3319, Val Loss: 30.6111
Epoch 30/50, Train Loss: 32.0186, Val Loss: 27.8222
Epoch 35/50, Train Loss: 35.3670, Val Loss: 29.7200
Epoch 40/50, Train Loss: 32.7869, Val Loss: 26.8910
Early stopping triggered after 41 epochs because no improvements over 10 subsequent epochs!
Epoch 41/50, Train Loss: 39.5270, Val Loss: 36.2600
The function took 80.66 seconds to execute.
No description has been provided for this image
Training Accuracy: 83.5735
Training Precision Scores: [17.88451712 99.03765187]
Training Recall Scores: [81.39534884 83.66869919]
Training F1 Scores: [29.3255132  90.70677023]

Validation Accuracy: 83.8006
Validation Precision Scores: [16.         99.18776875]
Validation Recall Scores: [81.72043011 83.87878788]
Validation F1 Scores: [26.76056338 90.89316988]

Threshold: 0.5
Validation Accuracy: 83.8006
Validation Precision Scores: [16.         99.18776875]
Validation Recall Scores: [81.72043011 83.87878788]
Validation F1 Scores: [26.76056338 90.89316988]


Threshold: 0.6
Validation Accuracy: 87.6947
Validation Precision Scores: [19.45205479 99.00136178]
Validation Recall Scores: [76.34408602 88.12121212]
Validation F1 Scores: [31.00436681 93.24497649]


Threshold: 0.7
Validation Accuracy: 91.8614
Validation Precision Scores: [26.2295082  98.75215146]
Validation Recall Scores: [68.8172043  92.72727273]
Validation F1 Scores: [37.98219585 95.64492603]


Threshold: 0.8
Validation Accuracy: 94.9766
Validation Precision Scores: [37.14285714 98.31136738]
Validation Recall Scores: [55.91397849 96.44444444]
Validation F1 Scores: [44.63519313 97.36895778]


Threshold: 0.9
Validation Accuracy: 96.8847
Validation Precision Scores: [60.31746032 97.80439122]
Validation Recall Scores: [40.86021505 98.98989899]
Validation F1 Scores: [48.71794872 98.3935743 ]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 256
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 64.0878, Val Loss: 43.8936
Epoch 10/50, Train Loss: 46.2565, Val Loss: 36.4003
Epoch 15/50, Train Loss: 58.5463, Val Loss: 42.2643
Early stopping triggered after 18 epochs because no improvements over 10 subsequent epochs!
Epoch 18/50, Train Loss: 49.8148, Val Loss: 50.8831
The function took 44.05 seconds to execute.
No description has been provided for this image
Training Accuracy: 84.2064
Training Precision Scores: [15.78645235 98.18245779]
Training Recall Scores: [63.95348837 85.09146341]
Training F1 Scores: [25.32228361 91.16942509]

Validation Accuracy: 84.3847
Validation Precision Scores: [14.67889908 98.63977486]
Validation Recall Scores: [68.8172043  84.96969697]
Validation F1 Scores: [24.19659735 91.29585414]

Threshold: 0.5
Validation Accuracy: 84.3847
Validation Precision Scores: [14.67889908 98.63977486]
Validation Recall Scores: [68.8172043  84.96969697]
Validation F1 Scores: [24.19659735 91.29585414]


Threshold: 0.6
Validation Accuracy: 89.7586
Validation Precision Scores: [18.04511278 98.04517811]
Validation Recall Scores: [51.61290323 91.19191919]
Validation F1 Scores: [26.74094708 94.49445259]


Threshold: 0.7
Validation Accuracy: 93.6916
Validation Precision Scores: [22.83464567 97.37812372]
Validation Recall Scores: [31.1827957  96.04040404]
Validation F1 Scores: [26.36363636 96.70463792]


Threshold: 0.8
Validation Accuracy: 95.6386
Validation Precision Scores: [31.37254902 96.94080254]
Validation Recall Scores: [17.20430108 98.58585859]
Validation F1 Scores: [22.22222222 97.75641026]


Threshold: 0.9
Validation Accuracy: 96.1059
Validation Precision Scores: [34.7826087  96.66011788]
Validation Recall Scores: [ 8.60215054 99.39393939]
Validation F1 Scores: [13.79310345 98.00796813]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 320
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 62.4660, Val Loss: 68.8774
Epoch 10/50, Train Loss: 58.6102, Val Loss: 49.6964
Epoch 15/50, Train Loss: 55.2249, Val Loss: 43.9672
Epoch 20/50, Train Loss: 46.3743, Val Loss: 37.5197
Epoch 25/50, Train Loss: 53.6285, Val Loss: 79.5858
Early stopping triggered after 30 epochs because no improvements over 10 subsequent epochs!
Epoch 30/50, Train Loss: 46.2963, Val Loss: 41.7887
The function took 83.69 seconds to execute.
No description has been provided for this image
Training Accuracy: 69.7274
Training Precision Scores: [10.33461652 98.82489482]
Training Recall Scores: [81.1627907  69.22764228]
Training F1 Scores: [18.3346467 81.4199486]

Validation Accuracy: 69.0421
Validation Precision Scores: [ 9.18604651 99.18032787]
Validation Recall Scores: [84.94623656 68.44444444]
Validation F1 Scores: [16.5792235  80.99450155]

Threshold: 0.5
Validation Accuracy: 69.0421
Validation Precision Scores: [ 9.18604651 99.18032787]
Validation Recall Scores: [84.94623656 68.44444444]
Validation F1 Scores: [16.5792235  80.99450155]


Threshold: 0.6
Validation Accuracy: 77.1028
Validation Precision Scores: [10.90047393 98.75968992]
Validation Recall Scores: [74.19354839 77.21212121]
Validation F1 Scores: [19.00826446 86.66666667]


Threshold: 0.7
Validation Accuracy: 85.2804
Validation Precision Scores: [12.5984252  97.94238683]
Validation Recall Scores: [51.61290323 86.54545455]
Validation F1 Scores: [20.25316456 91.89189189]


Threshold: 0.8
Validation Accuracy: 93.4190
Validation Precision Scores: [21.64179104 97.3705834 ]
Validation Recall Scores: [31.1827957  95.75757576]
Validation F1 Scores: [25.55066079 96.55734365]


Threshold: 0.9
Validation Accuracy: 95.7944
Validation Precision Scores: [31.70731707 96.83419074]
Validation Recall Scores: [13.97849462 98.86868687]
Validation F1 Scores: [19.40298507 97.84086365]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 384
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 63.3516, Val Loss: 69.9085
Epoch 10/50, Train Loss: 55.3681, Val Loss: 47.4163
Early stopping triggered after 11 epochs because no improvements over 10 subsequent epochs!
Epoch 11/50, Train Loss: 52.8863, Val Loss: 93.5456
The function took 33.76 seconds to execute.
No description has been provided for this image
Training Accuracy: 75.8325
Training Precision Scores: [ 9.47867299 97.54458516]
Training Recall Scores: [55.81395349 76.70731707]
Training F1 Scores: [16.20526671 85.88007737]

Validation Accuracy: 75.8567
Validation Precision Scores: [ 8.76369327 98.08190772]
Validation Recall Scores: [60.21505376 76.44444444]
Validation F1 Scores: [15.30054645 85.92188919]

Threshold: 0.5
Validation Accuracy: 75.8567
Validation Precision Scores: [ 8.76369327 98.08190772]
Validation Recall Scores: [60.21505376 76.44444444]
Validation F1 Scores: [15.30054645 85.92188919]


Threshold: 0.6
Validation Accuracy: 86.0981
Validation Precision Scores: [ 9.00621118 97.15048976]
Validation Recall Scores: [31.1827957  88.16161616]
Validation F1 Scores: [13.97590361 92.43804279]


Threshold: 0.7
Validation Accuracy: 93.1075
Validation Precision Scores: [12.5        96.78338762]
Validation Recall Scores: [15.05376344 96.04040404]
Validation F1 Scores: [13.65853659 96.41046441]


Threshold: 0.8
Validation Accuracy: 96.0670
Validation Precision Scores: [27.77777778 96.54901961]
Validation Recall Scores: [ 5.37634409 99.47474747]
Validation F1 Scores: [ 9.00900901 97.99004975]


Threshold: 0.9
Validation Accuracy: 96.3396
Validation Precision Scores: [40.         96.44947327]
Validation Recall Scores: [ 2.15053763 99.87878788]
Validation F1 Scores: [ 4.08163265 98.13418023]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 8 and hidden dim 512
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 65.7680, Val Loss: 51.6774
Epoch 10/50, Train Loss: 63.5539, Val Loss: 46.2024
Epoch 15/50, Train Loss: 77.2209, Val Loss: 50.1317
Early stopping triggered after 20 epochs because no improvements over 10 subsequent epochs!
Epoch 20/50, Train Loss: 109.7992, Val Loss: 54.4292
The function took 72.86 seconds to execute.
No description has been provided for this image
Training Accuracy: 80.9834
Training Precision Scores: [10.88854648 97.38075213]
Training Recall Scores: [49.30232558 82.36788618]
Training F1 Scores: [17.83761043 89.24737103]

Validation Accuracy: 80.7243
Validation Precision Scores: [10.11904762 97.96511628]
Validation Recall Scores: [54.83870968 81.6969697 ]
Validation F1 Scores: [17.08542714 89.09451421]

Threshold: 0.5
Validation Accuracy: 80.7243
Validation Precision Scores: [10.11904762 97.96511628]
Validation Recall Scores: [54.83870968 81.6969697 ]
Validation F1 Scores: [17.08542714 89.09451421]


Threshold: 0.6
Validation Accuracy: 90.1869
Validation Precision Scores: [12.67605634 97.19745223]
Validation Recall Scores: [29.03225806 92.48484848]
Validation F1 Scores: [17.64705882 94.7826087 ]


Threshold: 0.7
Validation Accuracy: 94.8988
Validation Precision Scores: [13.46153846 96.58187599]
Validation Recall Scores: [ 7.52688172 98.18181818]
Validation F1 Scores: [ 9.65517241 97.3752755 ]


Threshold: 0.8
Validation Accuracy: 96.0280
Validation Precision Scores: [23.52941176 96.51117209]
Validation Recall Scores: [ 4.30107527 99.47474747]
Validation F1 Scores: [ 7.27272727 97.97055312]


Threshold: 0.9
Validation Accuracy: 96.2227
Validation Precision Scores: [ 0.         96.37285491]
Validation Recall Scores: [ 0.         99.83838384]
Validation F1 Scores: [ 0.         98.07501488]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 16
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 34.1345, Val Loss: 31.5635
Epoch 10/50, Train Loss: 29.2327, Val Loss: 27.8675
Epoch 15/50, Train Loss: 26.7887, Val Loss: 29.0780
Epoch 20/50, Train Loss: 24.8438, Val Loss: 25.7108
Epoch 25/50, Train Loss: 24.8751, Val Loss: 26.0386
Epoch 30/50, Train Loss: 23.0415, Val Loss: 23.5316
Epoch 35/50, Train Loss: 20.7618, Val Loss: 22.3247
Epoch 40/50, Train Loss: 21.5685, Val Loss: 22.3910
Epoch 45/50, Train Loss: 21.1798, Val Loss: 25.2821
Epoch 50/50, Train Loss: 20.2502, Val Loss: 25.9048
The function took 87.52 seconds to execute.
No description has been provided for this image
Training Accuracy: 92.0643
Training Precision Scores: [33.00970874 99.38710737]
Training Recall Scores: [86.97674419 92.28658537]
Training F1 Scores: [47.85668586 95.7053275 ]

Validation Accuracy: 91.3162
Validation Precision Scores: [27.43055556 99.38596491]
Validation Recall Scores: [84.94623656 91.55555556]
Validation F1 Scores: [41.46981627 95.31019979]

Threshold: 0.5
Validation Accuracy: 91.3162
Validation Precision Scores: [27.43055556 99.38596491]
Validation Recall Scores: [84.94623656 91.55555556]
Validation F1 Scores: [41.46981627 95.31019979]


Threshold: 0.6
Validation Accuracy: 93.2243
Validation Precision Scores: [32.46753247 99.22978177]
Validation Recall Scores: [80.64516129 93.6969697 ]
Validation F1 Scores: [46.2962963 96.3840399]


Threshold: 0.7
Validation Accuracy: 94.7430
Validation Precision Scores: [38.20224719 98.9539749 ]
Validation Recall Scores: [73.11827957 95.55555556]
Validation F1 Scores: [50.18450185 97.22507708]


Threshold: 0.8
Validation Accuracy: 96.0670
Validation Precision Scores: [46.875      98.64754098]
Validation Recall Scores: [64.51612903 97.25252525]
Validation F1 Scores: [54.29864253 97.94506612]


Threshold: 0.9
Validation Accuracy: 96.4953
Validation Precision Scores: [51.80722892 97.98792757]
Validation Recall Scores: [46.23655914 98.38383838]
Validation F1 Scores: [48.86363636 98.18548387]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 32
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 37.2835, Val Loss: 33.8163
Epoch 10/50, Train Loss: 32.6587, Val Loss: 35.4304
Epoch 15/50, Train Loss: 30.8448, Val Loss: 26.6918
Epoch 20/50, Train Loss: 29.3204, Val Loss: 28.6762
Early stopping triggered after 25 epochs because no improvements over 10 subsequent epochs!
Epoch 25/50, Train Loss: 27.3044, Val Loss: 27.7777
The function took 44.15 seconds to execute.
No description has been provided for this image
Training Accuracy: 83.0088
Training Precision Scores: [17.56290084 99.10226859]
Training Recall Scores: [82.79069767 83.01829268]
Training F1 Scores: [28.97842898 90.35005254]

Validation Accuracy: 83.5670
Validation Precision Scores: [16.22176591 99.32724652]
Validation Recall Scores: [84.94623656 83.51515152]
Validation F1 Scores: [27.24137931 90.73748903]

Threshold: 0.5
Validation Accuracy: 83.5670
Validation Precision Scores: [16.22176591 99.32724652]
Validation Recall Scores: [84.94623656 83.51515152]
Validation F1 Scores: [27.24137931 90.73748903]


Threshold: 0.6
Validation Accuracy: 89.1745
Validation Precision Scores: [21.71253823 99.0182954 ]
Validation Recall Scores: [76.34408602 89.65656566]
Validation F1 Scores: [33.80952381 94.10517388]


Threshold: 0.7
Validation Accuracy: 93.1854
Validation Precision Scores: [29.29292929 98.52320675]
Validation Recall Scores: [62.3655914  94.34343434]
Validation F1 Scores: [39.86254296 96.3880289 ]


Threshold: 0.8
Validation Accuracy: 95.6776
Validation Precision Scores: [41.96428571 98.12703583]
Validation Recall Scores: [50.53763441 97.37373737]
Validation F1 Scores: [45.85365854 97.74893531]


Threshold: 0.9
Validation Accuracy: 96.4564
Validation Precision Scores: [51.66666667 97.52791069]
Validation Recall Scores: [33.33333333 98.82828283]
Validation F1 Scores: [40.52287582 98.17379089]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 64
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 40.8258, Val Loss: 34.7488
Epoch 10/50, Train Loss: 30.4454, Val Loss: 32.1898
Epoch 15/50, Train Loss: 27.7552, Val Loss: 27.7033
Epoch 20/50, Train Loss: 27.8521, Val Loss: 26.0819
Epoch 25/50, Train Loss: 25.2240, Val Loss: 26.1282
Epoch 30/50, Train Loss: 23.3502, Val Loss: 24.5768
Early stopping triggered after 33 epochs because no improvements over 10 subsequent epochs!
Epoch 33/50, Train Loss: 23.5237, Val Loss: 26.7141
The function took 60.21 seconds to execute.
No description has been provided for this image
Training Accuracy: 85.5794
Training Precision Scores: [21.64058284 99.65545919]
Training Recall Scores: [93.25581395 85.24390244]
Training F1 Scores: [35.12921594 91.88804294]

Validation Accuracy: 85.6308
Validation Precision Scores: [18.63636364 99.48308271]
Validation Recall Scores: [88.17204301 85.53535354]
Validation F1 Scores: [30.76923077 91.98348903]

Threshold: 0.5
Validation Accuracy: 85.6308
Validation Precision Scores: [18.63636364 99.48308271]
Validation Recall Scores: [88.17204301 85.53535354]
Validation F1 Scores: [30.76923077 91.98348903]


Threshold: 0.6
Validation Accuracy: 89.4860
Validation Precision Scores: [23.7388724  99.41730166]
Validation Recall Scores: [86.02150538 89.61616162]
Validation F1 Scores: [37.20930233 94.26264343]


Threshold: 0.7
Validation Accuracy: 92.7570
Validation Precision Scores: [30.54393305 99.14126234]
Validation Recall Scores: [78.49462366 93.29292929]
Validation F1 Scores: [43.97590361 96.12822648]


Threshold: 0.8
Validation Accuracy: 94.7819
Validation Precision Scores: [37.26708075 98.62899875]
Validation Recall Scores: [64.51612903 95.91919192]
Validation F1 Scores: [47.24409449 97.25522327]


Threshold: 0.9
Validation Accuracy: 96.4564
Validation Precision Scores: [51.02040816 98.25910931]
Validation Recall Scores: [53.76344086 98.06060606]
Validation F1 Scores: [52.35602094 98.15975733]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 128
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 40.4202, Val Loss: 32.9761
Epoch 10/50, Train Loss: 35.3946, Val Loss: 33.1016
Epoch 15/50, Train Loss: 32.8148, Val Loss: 40.8753
Epoch 20/50, Train Loss: 37.1091, Val Loss: 35.9317
Early stopping triggered after 24 epochs because no improvements over 10 subsequent epochs!
Epoch 24/50, Train Loss: 34.8422, Val Loss: 28.5075
The function took 48.92 seconds to execute.
No description has been provided for this image
Training Accuracy: 83.2717
Training Precision Scores: [17.8        99.10519952]
Training Recall Scores: [82.79069767 83.29268293]
Training F1 Scores: [29.30041152 90.51352844]

Validation Accuracy: 83.5670
Validation Precision Scores: [16.63286004 99.46987952]
Validation Recall Scores: [88.17204301 83.39393939]
Validation F1 Scores: [27.98634812 90.72527473]

Threshold: 0.5
Validation Accuracy: 83.5670
Validation Precision Scores: [16.63286004 99.46987952]
Validation Recall Scores: [88.17204301 83.39393939]
Validation F1 Scores: [27.98634812 90.72527473]


Threshold: 0.6
Validation Accuracy: 88.3178
Validation Precision Scores: [21.32963989 99.27503398]
Validation Recall Scores: [82.79569892 88.52525253]
Validation F1 Scores: [33.92070485 93.59248185]


Threshold: 0.7
Validation Accuracy: 92.1340
Validation Precision Scores: [26.80851064 98.71410201]
Validation Recall Scores: [67.74193548 93.05050505]
Validation F1 Scores: [38.41463415 95.79866889]


Threshold: 0.8
Validation Accuracy: 93.9642
Validation Precision Scores: [30.12820513 98.09286899]
Validation Recall Scores: [50.53763441 95.5959596 ]
Validation F1 Scores: [37.75100402 96.82832003]


Threshold: 0.9
Validation Accuracy: 95.5218
Validation Precision Scores: [39.42307692 97.88961039]
Validation Recall Scores: [44.08602151 97.45454545]
Validation F1 Scores: [41.62436548 97.67159344]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 256
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 49.9153, Val Loss: 64.4564
Epoch 10/50, Train Loss: 48.0615, Val Loss: 56.3368
Epoch 15/50, Train Loss: 65.9540, Val Loss: 54.8457
Early stopping triggered after 16 epochs because no improvements over 10 subsequent epochs!
Epoch 16/50, Train Loss: 51.4240, Val Loss: 61.7117
The function took 39.62 seconds to execute.
No description has been provided for this image
Training Accuracy: 83.9435
Training Precision Scores: [16.30735213 98.40444392]
Training Recall Scores: [68.60465116 84.61382114]
Training F1 Scores: [26.35104958 90.98956341]

Validation Accuracy: 82.5156
Validation Precision Scores: [14.54183267 99.03194579]
Validation Recall Scores: [78.49462366 82.66666667]
Validation F1 Scores: [24.53781513 90.11231006]

Threshold: 0.5
Validation Accuracy: 82.5156
Validation Precision Scores: [14.54183267 99.03194579]
Validation Recall Scores: [78.49462366 82.66666667]
Validation F1 Scores: [24.53781513 90.11231006]


Threshold: 0.6
Validation Accuracy: 91.6277
Validation Precision Scores: [24.3697479  98.49785408]
Validation Recall Scores: [62.3655914  92.72727273]
Validation F1 Scores: [35.04531722 95.52549428]


Threshold: 0.7
Validation Accuracy: 94.5872
Validation Precision Scores: [31.4516129  97.79050736]
Validation Recall Scores: [41.93548387 96.56565657]
Validation F1 Scores: [35.94470046 97.1742224 ]


Threshold: 0.8
Validation Accuracy: 95.5607
Validation Precision Scores: [38.46153846 97.65845781]
Validation Recall Scores: [37.6344086  97.73737374]
Validation F1 Scores: [38.04347826 97.69789984]


Threshold: 0.9
Validation Accuracy: 95.9891
Validation Precision Scores: [40.38461538 97.13831479]
Validation Recall Scores: [22.58064516 98.74747475]
Validation F1 Scores: [28.96551724 97.93628531]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 320
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 62.7902, Val Loss: 69.1210
Epoch 10/50, Train Loss: 103.4131, Val Loss: 75.1006
Epoch 15/50, Train Loss: 44.4062, Val Loss: 45.0601
Early stopping triggered after 16 epochs because no improvements over 10 subsequent epochs!
Epoch 16/50, Train Loss: 47.5345, Val Loss: 41.9938
The function took 45.24 seconds to execute.
No description has been provided for this image
Training Accuracy: 83.4372
Training Precision Scores: [17.15762274 98.82423515]
Training Recall Scores: [77.20930233 83.70934959]
Training F1 Scores: [28.07610994 90.64099037]

Validation Accuracy: 83.7617
Validation Precision Scores: [16.39004149 99.32885906]
Validation Recall Scores: [84.94623656 83.71717172]
Validation F1 Scores: [27.47826087 90.85726814]

Threshold: 0.5
Validation Accuracy: 83.7617
Validation Precision Scores: [16.39004149 99.32885906]
Validation Recall Scores: [84.94623656 83.71717172]
Validation F1 Scores: [27.47826087 90.85726814]


Threshold: 0.6
Validation Accuracy: 88.7850
Validation Precision Scores: [20.18348624 98.79518072]
Validation Recall Scores: [70.96774194 89.45454545]
Validation F1 Scores: [31.42857143 93.89312977]


Threshold: 0.7
Validation Accuracy: 91.3941
Validation Precision Scores: [23.55371901 98.45227859]
Validation Recall Scores: [61.29032258 92.52525253]
Validation F1 Scores: [34.02985075 95.39679233]


Threshold: 0.8
Validation Accuracy: 93.7305
Validation Precision Scores: [27.92207792 97.92874896]
Validation Recall Scores: [46.23655914 95.51515152]
Validation F1 Scores: [34.81781377 96.70689303]


Threshold: 0.9
Validation Accuracy: 94.9377
Validation Precision Scores: [31.68316832 97.52736117]
Validation Recall Scores: [34.40860215 97.21212121]
Validation F1 Scores: [32.98969072 97.36948604]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 384
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 63.4676, Val Loss: 34.8579
Epoch 10/50, Train Loss: 46.2697, Val Loss: 40.7314
Epoch 15/50, Train Loss: 61.5719, Val Loss: 43.2691
Epoch 20/50, Train Loss: 54.5525, Val Loss: 56.6257
Early stopping triggered after 24 epochs because no improvements over 10 subsequent epochs!
Epoch 24/50, Train Loss: 45.1673, Val Loss: 37.7554
The function took 74.77 seconds to execute.
No description has been provided for this image
Training Accuracy: 89.1431
Training Precision Scores: [22.70916335 98.39156961]
Training Recall Scores: [66.27906977 90.14227642]
Training F1 Scores: [33.82789318 94.08644922]

Validation Accuracy: 89.7975
Validation Precision Scores: [20.55749129 98.50942569]
Validation Recall Scores: [63.44086022 90.78787879]
Validation F1 Scores: [31.05263158 94.49116905]

Threshold: 0.5
Validation Accuracy: 89.7975
Validation Precision Scores: [20.55749129 98.50942569]
Validation Recall Scores: [63.44086022 90.78787879]
Validation F1 Scores: [31.05263158 94.49116905]


Threshold: 0.6
Validation Accuracy: 92.4455
Validation Precision Scores: [26.06635071 98.38778108]
Validation Recall Scores: [59.13978495 93.6969697 ]
Validation F1 Scores: [36.18421053 95.98509934]


Threshold: 0.7
Validation Accuracy: 94.3925
Validation Precision Scores: [31.91489362 98.02224969]
Validation Recall Scores: [48.38709677 96.12121212]
Validation F1 Scores: [38.46153846 97.0624235 ]


Threshold: 0.8
Validation Accuracy: 95.6386
Validation Precision Scores: [40.20618557 97.81464994]
Validation Recall Scores: [41.93548387 97.65656566]
Validation F1 Scores: [41.05263158 97.73554387]


Threshold: 0.9
Validation Accuracy: 96.3785
Validation Precision Scores: [50.        97.4501992]
Validation Recall Scores: [31.1827957  98.82828283]
Validation F1 Scores: [38.41059603 98.13440321]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 16 and hidden dim 512
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 60.3527, Val Loss: 49.3524
Epoch 10/50, Train Loss: 52.6831, Val Loss: 61.1173
Epoch 15/50, Train Loss: 56.0051, Val Loss: 38.3192
Epoch 20/50, Train Loss: 101.9491, Val Loss: 53.2053
Early stopping triggered after 25 epochs because no improvements over 10 subsequent epochs!
Epoch 25/50, Train Loss: 49.5788, Val Loss: 47.0921
The function took 92.87 seconds to execute.
No description has been provided for this image
Training Accuracy: 78.5394
Training Precision Scores: [13.40759076 98.66173847]
Training Recall Scores: [75.58139535 78.66869919]
Training F1 Scores: [22.77505256 87.53816578]

Validation Accuracy: 77.6090
Validation Precision Scores: [10.87662338 98.66803279]
Validation Recall Scores: [72.04301075 77.81818182]
Validation F1 Scores: [18.89985896 87.01152022]

Threshold: 0.5
Validation Accuracy: 77.6090
Validation Precision Scores: [10.87662338 98.66803279]
Validation Recall Scores: [72.04301075 77.81818182]
Validation F1 Scores: [18.89985896 87.01152022]


Threshold: 0.6
Validation Accuracy: 84.8131
Validation Precision Scores: [13.15136476 98.15242494]
Validation Recall Scores: [56.98924731 85.85858586]
Validation F1 Scores: [21.37096774 91.59482759]


Threshold: 0.7
Validation Accuracy: 90.4595
Validation Precision Scores: [16.95652174 97.69033362]
Validation Recall Scores: [41.93548387 92.28282828]
Validation F1 Scores: [24.14860681 94.90961978]


Threshold: 0.8
Validation Accuracy: 93.5748
Validation Precision Scores: [20.         97.18137255]
Validation Recall Scores: [25.80645161 96.12121212]
Validation F1 Scores: [22.53521127 96.64838513]


Threshold: 0.9
Validation Accuracy: 95.5997
Validation Precision Scores: [32.75862069 97.05179283]
Validation Recall Scores: [20.43010753 98.42424242]
Validation F1 Scores: [25.16556291 97.7331996 ]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 32
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 33.7671, Val Loss: 33.8390
Epoch 10/50, Train Loss: 29.6380, Val Loss: 28.0906
Epoch 15/50, Train Loss: 27.4501, Val Loss: 27.4199
Epoch 20/50, Train Loss: 26.7367, Val Loss: 32.8043
Early stopping triggered after 24 epochs because no improvements over 10 subsequent epochs!
Epoch 24/50, Train Loss: 25.1395, Val Loss: 30.5434
The function took 43.50 seconds to execute.
No description has been provided for this image
Training Accuracy: 84.5083
Training Precision Scores: [19.87545407 99.43665348]
Training Recall Scores: [89.06976744 84.30894309]
Training F1 Scores: [32.49893933 91.25006875]

Validation Accuracy: 84.3458
Validation Precision Scores: [17.74530271 99.61704165]
Validation Recall Scores: [91.39784946 84.08080808]
Validation F1 Scores: [29.72027972 91.1919369 ]

Threshold: 0.5
Validation Accuracy: 84.3458
Validation Precision Scores: [17.74530271 99.61704165]
Validation Recall Scores: [91.39784946 84.08080808]
Validation F1 Scores: [29.72027972 91.1919369 ]


Threshold: 0.6
Validation Accuracy: 89.2913
Validation Precision Scores: [23.39181287 99.41599281]
Validation Recall Scores: [86.02150538 89.41414141]
Validation F1 Scores: [36.7816092  94.15018081]


Threshold: 0.7
Validation Accuracy: 93.3801
Validation Precision Scores: [32.88888889 99.18907384]
Validation Recall Scores: [79.56989247 93.8989899 ]
Validation F1 Scores: [46.5408805  96.47156496]


Threshold: 0.8
Validation Accuracy: 95.0935
Validation Precision Scores: [38.9261745  98.55312112]
Validation Recall Scores: [62.3655914  96.32323232]
Validation F1 Scores: [47.9338843  97.42541888]


Threshold: 0.9
Validation Accuracy: 96.2617
Validation Precision Scores: [47.88732394 97.6371646 ]
Validation Recall Scores: [36.55913978 98.50505051]
Validation F1 Scores: [41.46341463 98.06918745]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 64
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 34.8468, Val Loss: 33.7676
Epoch 10/50, Train Loss: 32.1142, Val Loss: 28.4629
Epoch 15/50, Train Loss: 30.3931, Val Loss: 32.0379
Epoch 20/50, Train Loss: 25.5999, Val Loss: 23.4243
Epoch 25/50, Train Loss: 25.0089, Val Loss: 26.0679
Early stopping triggered after 30 epochs because no improvements over 10 subsequent epochs!
Epoch 30/50, Train Loss: 27.9731, Val Loss: 37.8336
The function took 56.55 seconds to execute.
No description has been provided for this image
Training Accuracy: 87.4294
Training Precision Scores: [23.24425109 99.35342339]
Training Recall Scores: [86.97674419 87.44918699]
Training F1 Scores: [36.68464934 93.02199881]

Validation Accuracy: 87.9673
Validation Precision Scores: [21.57894737 99.49725777]
Validation Recall Scores: [88.17204301 87.95959596]
Validation F1 Scores: [34.67230444 93.37336479]

Threshold: 0.5
Validation Accuracy: 87.9673
Validation Precision Scores: [21.57894737 99.49725777]
Validation Recall Scores: [88.17204301 87.95959596]
Validation F1 Scores: [34.67230444 93.37336479]


Threshold: 0.6
Validation Accuracy: 91.0826
Validation Precision Scores: [26.55172414 99.2976295 ]
Validation Recall Scores: [82.79569892 91.39393939]
Validation F1 Scores: [40.20887728 95.18199032]


Threshold: 0.7
Validation Accuracy: 93.2243
Validation Precision Scores: [32.46753247 99.22978177]
Validation Recall Scores: [80.64516129 93.6969697 ]
Validation F1 Scores: [46.2962963 96.3840399]


Threshold: 0.8
Validation Accuracy: 94.8209
Validation Precision Scores: [38.23529412 98.8323603 ]
Validation Recall Scores: [69.89247312 95.75757576]
Validation F1 Scores: [49.42965779 97.27067515]


Threshold: 0.9
Validation Accuracy: 96.3396
Validation Precision Scores: [49.47368421 98.13991104]
Validation Recall Scores: [50.53763441 98.06060606]
Validation F1 Scores: [50.         98.10024252]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 128
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 39.8849, Val Loss: 39.8646
Epoch 10/50, Train Loss: 35.5747, Val Loss: 37.6756
Epoch 15/50, Train Loss: 30.7568, Val Loss: 29.1598
Epoch 20/50, Train Loss: 31.6825, Val Loss: 28.2356
Early stopping triggered after 24 epochs because no improvements over 10 subsequent epochs!
Epoch 24/50, Train Loss: 30.2242, Val Loss: 28.3046
The function took 49.87 seconds to execute.
No description has been provided for this image
Training Accuracy: 82.9990
Training Precision Scores: [17.80821918 99.19766594]
Training Recall Scores: [84.65116279 82.92682927]
Training F1 Scores: [29.42603072 90.33543673]

Validation Accuracy: 82.9439
Validation Precision Scores: [15.70576541 99.3220339 ]
Validation Recall Scores: [84.94623656 82.86868687]
Validation F1 Scores: [26.51006711 90.35242291]

Threshold: 0.5
Validation Accuracy: 82.9439
Validation Precision Scores: [15.70576541 99.3220339 ]
Validation Recall Scores: [84.94623656 82.86868687]
Validation F1 Scores: [26.51006711 90.35242291]


Threshold: 0.6
Validation Accuracy: 87.2274
Validation Precision Scores: [18.99736148 99.04065783]
Validation Recall Scores: [77.41935484 87.5959596 ]
Validation F1 Scores: [30.50847458 92.96740995]


Threshold: 0.7
Validation Accuracy: 92.2118
Validation Precision Scores: [27.80082988 98.88268156]
Validation Recall Scores: [72.04301075 92.96969697]
Validation F1 Scores: [40.11976048 95.83506872]


Threshold: 0.8
Validation Accuracy: 93.9252
Validation Precision Scores: [31.36094675 98.3326386 ]
Validation Recall Scores: [56.98924731 95.31313131]
Validation F1 Scores: [40.45801527 96.79934346]


Threshold: 0.9
Validation Accuracy: 96.0280
Validation Precision Scores: [45.54455446 98.09485205]
Validation Recall Scores: [49.46236559 97.77777778]
Validation F1 Scores: [47.42268041 97.93605828]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 256
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 50.4116, Val Loss: 39.0281
Epoch 10/50, Train Loss: 42.8541, Val Loss: 38.5277
Epoch 15/50, Train Loss: 41.0493, Val Loss: 44.8510
Epoch 20/50, Train Loss: 52.8881, Val Loss: 44.6988
Early stopping triggered after 21 epochs because no improvements over 10 subsequent epochs!
Epoch 21/50, Train Loss: 39.7932, Val Loss: 34.5731
The function took 53.61 seconds to execute.
No description has been provided for this image
Training Accuracy: 86.8160
Training Precision Scores: [20.90680101 98.87122783]
Training Recall Scores: [77.20930233 87.23577236]
Training F1 Scores: [32.90386521 92.68977432]

Validation Accuracy: 87.0717
Validation Precision Scores: [19.43734015 99.21910887]
Validation Recall Scores: [81.72043011 87.27272727]
Validation F1 Scores: [31.40495868 92.86328461]

Threshold: 0.5
Validation Accuracy: 87.0717
Validation Precision Scores: [19.43734015 99.21910887]
Validation Recall Scores: [81.72043011 87.27272727]
Validation F1 Scores: [31.40495868 92.86328461]


Threshold: 0.6
Validation Accuracy: 90.6931
Validation Precision Scores: [23.35766423 98.73583261]
Validation Recall Scores: [68.8172043  91.51515152]
Validation F1 Scores: [34.8773842  94.98846718]


Threshold: 0.7
Validation Accuracy: 92.8349
Validation Precision Scores: [24.58100559 97.94893261]
Validation Recall Scores: [47.31182796 94.54545455]
Validation F1 Scores: [32.35294118 96.21710526]


Threshold: 0.8
Validation Accuracy: 94.5872
Validation Precision Scores: [30.50847458 97.67346939]
Validation Recall Scores: [38.70967742 96.68686869]
Validation F1 Scores: [34.12322275 97.17766497]


Threshold: 0.9
Validation Accuracy: 95.9502
Validation Precision Scores: [41.26984127 97.3253493 ]
Validation Recall Scores: [27.95698925 98.50505051]
Validation F1 Scores: [33.33333333 97.91164659]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 320
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 57.1849, Val Loss: 45.5530
Epoch 10/50, Train Loss: 44.0882, Val Loss: 29.7315
Epoch 15/50, Train Loss: 45.4767, Val Loss: 34.4734
Early stopping triggered after 20 epochs because no improvements over 10 subsequent epochs!
Epoch 20/50, Train Loss: 51.8350, Val Loss: 42.3385
The function took 57.40 seconds to execute.
No description has been provided for this image
Training Accuracy: 89.7371
Training Precision Scores: [23.64864865 98.34910852]
Training Recall Scores: [65.11627907 90.81300813]
Training F1 Scores: [34.69640644 94.43094156]

Validation Accuracy: 90.3037
Validation Precision Scores: [23.82550336 99.030837  ]
Validation Recall Scores: [76.34408602 90.82828283]
Validation F1 Scores: [36.31713555 94.75237092]

Threshold: 0.5
Validation Accuracy: 90.3037
Validation Precision Scores: [23.82550336 99.030837  ]
Validation Recall Scores: [76.34408602 90.82828283]
Validation F1 Scores: [36.31713555 94.75237092]


Threshold: 0.6
Validation Accuracy: 92.3287
Validation Precision Scores: [26.78571429 98.59215017]
Validation Recall Scores: [64.51612903 93.37373737]
Validation F1 Scores: [37.85488959 95.91201494]


Threshold: 0.7
Validation Accuracy: 94.1589
Validation Precision Scores: [32.51533742 98.33679834]
Validation Recall Scores: [56.98924731 95.55555556]
Validation F1 Scores: [41.40625    96.92622951]


Threshold: 0.8
Validation Accuracy: 95.5997
Validation Precision Scores: [40.38461538 97.93019481]
Validation Recall Scores: [45.16129032 97.49494949]
Validation F1 Scores: [42.63959391 97.71208747]


Threshold: 0.9
Validation Accuracy: 96.5732
Validation Precision Scores: [53.52112676 97.79735683]
Validation Recall Scores: [40.86021505 98.66666667]
Validation F1 Scores: [46.34146341 98.2300885 ]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 384
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 55.7635, Val Loss: 44.7936
Epoch 10/50, Train Loss: 40.8467, Val Loss: 38.7768
Epoch 15/50, Train Loss: 44.5060, Val Loss: 52.2695
Epoch 20/50, Train Loss: 77.0556, Val Loss: 35.7570
Epoch 25/50, Train Loss: 52.6911, Val Loss: 38.1255
Epoch 30/50, Train Loss: 49.3987, Val Loss: 37.6152
Early stopping triggered after 33 epochs because no improvements over 10 subsequent epochs!
Epoch 33/50, Train Loss: 46.4249, Val Loss: 49.6761
The function took 104.80 seconds to execute.
No description has been provided for this image
Training Accuracy: 81.4606
Training Precision Scores: [16.37773723 99.12106957]
Training Recall Scores: [83.48837209 81.37195122]
Training F1 Scores: [27.38367658 89.37381404]

Validation Accuracy: 80.1012
Validation Precision Scores: [13.96551724 99.39637827]
Validation Recall Scores: [87.09677419 79.83838384]
Validation F1 Scores: [24.07132244 88.55030249]

Threshold: 0.5
Validation Accuracy: 80.1012
Validation Precision Scores: [13.96551724 99.39637827]
Validation Recall Scores: [87.09677419 79.83838384]
Validation F1 Scores: [24.07132244 88.55030249]


Threshold: 0.6
Validation Accuracy: 85.9034
Validation Precision Scores: [17.27493917 98.9800649 ]
Validation Recall Scores: [76.34408602 86.26262626]
Validation F1 Scores: [28.17460317 92.18480138]


Threshold: 0.7
Validation Accuracy: 90.2648
Validation Precision Scores: [20.81784387 98.39060461]
Validation Recall Scores: [60.21505376 91.39393939]
Validation F1 Scores: [30.93922652 94.76330121]


Threshold: 0.8
Validation Accuracy: 92.9128
Validation Precision Scores: [26.20320856 98.15203696]
Validation Recall Scores: [52.68817204 94.42424242]
Validation F1 Scores: [35.         96.25205931]


Threshold: 0.9
Validation Accuracy: 94.8988
Validation Precision Scores: [34.92063492 97.99344799]
Validation Recall Scores: [47.31182796 96.68686869]
Validation F1 Scores: [40.1826484  97.33577385]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 32 and hidden dim 512
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 55.8639, Val Loss: 49.1705
Epoch 10/50, Train Loss: 48.8122, Val Loss: 117.4936
Epoch 15/50, Train Loss: 58.5856, Val Loss: 37.8413
Epoch 20/50, Train Loss: 47.2577, Val Loss: 33.6201
Epoch 25/50, Train Loss: 62.4749, Val Loss: 47.0653
Early stopping triggered after 30 epochs because no improvements over 10 subsequent epochs!
Epoch 30/50, Train Loss: 56.4464, Val Loss: 80.3027
The function took 112.90 seconds to execute.
No description has been provided for this image
Training Accuracy: 89.4937
Training Precision Scores: [22.61603376 98.21684095]
Training Recall Scores: [62.3255814  90.68089431]
Training F1 Scores: [33.18885449 94.2985469 ]

Validation Accuracy: 88.2788
Validation Precision Scores: [17.29559748 98.31111111]
Validation Recall Scores: [59.13978495 89.37373737]
Validation F1 Scores: [26.76399027 93.62962963]

Threshold: 0.5
Validation Accuracy: 88.2788
Validation Precision Scores: [17.29559748 98.31111111]
Validation Recall Scores: [59.13978495 89.37373737]
Validation F1 Scores: [26.76399027 93.62962963]


Threshold: 0.6
Validation Accuracy: 91.0047
Validation Precision Scores: [20.51282051 98.07197943]
Validation Recall Scores: [51.61290323 92.48484848]
Validation F1 Scores: [29.35779817 95.19650655]


Threshold: 0.7
Validation Accuracy: 92.6791
Validation Precision Scores: [24.04371585 97.94549266]
Validation Recall Scores: [47.31182796 94.38383838]
Validation F1 Scores: [31.88405797 96.13168724]


Threshold: 0.8
Validation Accuracy: 94.1978
Validation Precision Scores: [27.77777778 97.62489762]
Validation Recall Scores: [37.6344086  96.32323232]
Validation F1 Scores: [31.96347032 96.96969697]


Threshold: 0.9
Validation Accuracy: 95.7555
Validation Precision Scores: [37.87878788 97.28217426]
Validation Recall Scores: [26.88172043 98.34343434]
Validation F1 Scores: [31.44654088 97.80992566]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 64
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 35.2130, Val Loss: 44.7467
Epoch 10/50, Train Loss: 29.0355, Val Loss: 29.7030
Epoch 15/50, Train Loss: 29.2398, Val Loss: 40.4835
Early stopping triggered after 19 epochs because no improvements over 10 subsequent epochs!
Epoch 19/50, Train Loss: 27.9417, Val Loss: 37.7175
The function took 37.93 seconds to execute.
No description has been provided for this image
Training Accuracy: 88.7926
Training Precision Scores: [24.45074415 99.04052376]
Training Recall Scores: [80.23255814 89.16666667]
Training F1 Scores: [37.47963064 93.84459062]

Validation Accuracy: 88.2399
Validation Precision Scores: [20.22792023 99.00766802]
Validation Recall Scores: [76.34408602 88.68686869]
Validation F1 Scores: [31.98198198 93.56351236]

Threshold: 0.5
Validation Accuracy: 88.2399
Validation Precision Scores: [20.22792023 99.00766802]
Validation Recall Scores: [76.34408602 88.68686869]
Validation F1 Scores: [31.98198198 93.56351236]


Threshold: 0.6
Validation Accuracy: 91.6667
Validation Precision Scores: [26.08695652 98.8336933 ]
Validation Recall Scores: [70.96774194 92.44444444]
Validation F1 Scores: [38.15028902 95.53235908]


Threshold: 0.7
Validation Accuracy: 94.3146
Validation Precision Scores: [34.50292398 98.58156028]
Validation Recall Scores: [63.44086022 95.47474747]
Validation F1 Scores: [44.6969697  97.00328407]


Threshold: 0.8
Validation Accuracy: 95.2492
Validation Precision Scores: [38.21138211 98.11860941]
Validation Recall Scores: [50.53763441 96.92929293]
Validation F1 Scores: [43.51851852 97.5203252 ]


Threshold: 0.9
Validation Accuracy: 96.4953
Validation Precision Scores: [51.80722892 97.98792757]
Validation Recall Scores: [46.23655914 98.38383838]
Validation F1 Scores: [48.86363636 98.18548387]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 128
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 37.4374, Val Loss: 42.2310
Epoch 10/50, Train Loss: 34.8094, Val Loss: 39.3751
Epoch 15/50, Train Loss: 35.0429, Val Loss: 31.7742
Epoch 20/50, Train Loss: 33.0401, Val Loss: 34.9874
Epoch 25/50, Train Loss: 33.2939, Val Loss: 36.5560
Early stopping triggered after 26 epochs because no improvements over 10 subsequent epochs!
Epoch 26/50, Train Loss: 32.1226, Val Loss: 54.0640
The function took 57.62 seconds to execute.
No description has been provided for this image
Training Accuracy: 78.4615
Training Precision Scores: [15.35769829 99.54533645]
Training Recall Scores: [91.86046512 77.87601626]
Training F1 Scores: [26.31578947 87.38738739]

Validation Accuracy: 77.6480
Validation Precision Scores: [12.94298921 99.53100573]
Validation Recall Scores: [90.32258065 77.17171717]
Validation F1 Scores: [22.64150943 86.93673191]

Threshold: 0.5
Validation Accuracy: 77.6480
Validation Precision Scores: [12.94298921 99.53100573]
Validation Recall Scores: [90.32258065 77.17171717]
Validation F1 Scores: [22.64150943 86.93673191]


Threshold: 0.6
Validation Accuracy: 83.4502
Validation Precision Scores: [16.2601626  99.37379576]
Validation Recall Scores: [86.02150538 83.35353535]
Validation F1 Scores: [27.35042735 90.6613931 ]


Threshold: 0.7
Validation Accuracy: 88.5903
Validation Precision Scores: [21.75141243 99.27732611]
Validation Recall Scores: [82.79569892 88.80808081]
Validation F1 Scores: [34.45190157 93.75133291]


Threshold: 0.8
Validation Accuracy: 92.4844
Validation Precision Scores: [27.87610619 98.71904355]
Validation Recall Scores: [67.74193548 93.41414141]
Validation F1 Scores: [39.4984326  95.99335686]


Threshold: 0.9
Validation Accuracy: 94.4315
Validation Precision Scores: [31.88405797 97.98353909]
Validation Recall Scores: [47.31182796 96.2020202 ]
Validation F1 Scores: [38.0952381  97.08460754]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 256
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 48.7994, Val Loss: 53.0937
Epoch 10/50, Train Loss: 44.3505, Val Loss: 55.3668
Early stopping triggered after 12 epochs because no improvements over 10 subsequent epochs!
Epoch 12/50, Train Loss: 44.6267, Val Loss: 542.6777
The function took 32.30 seconds to execute.
No description has been provided for this image
Training Accuracy: 75.8617
Training Precision Scores: [12.29297019 98.72898186]
Training Recall Scores: [77.6744186  75.78252033]
Training F1 Scores: [21.22656498 85.74713965]

Validation Accuracy: 73.7539
Validation Precision Scores: [ 9.70873786 98.75473741]
Validation Recall Scores: [75.2688172 73.6969697]
Validation F1 Scores: [17.1990172  84.40536789]

Threshold: 0.5
Validation Accuracy: 73.7539
Validation Precision Scores: [ 9.70873786 98.75473741]
Validation Recall Scores: [75.2688172 73.6969697]
Validation F1 Scores: [17.1990172  84.40536789]


Threshold: 0.6
Validation Accuracy: 81.1916
Validation Precision Scores: [11.76470588 98.39650146]
Validation Recall Scores: [64.51612903 81.81818182]
Validation F1 Scores: [19.90049751 89.34480477]


Threshold: 0.7
Validation Accuracy: 87.5000
Validation Precision Scores: [15.24390244 98.08035714]
Validation Recall Scores: [53.76344086 88.76767677]
Validation F1 Scores: [23.75296912 93.19194062]


Threshold: 0.8
Validation Accuracy: 91.4330
Validation Precision Scores: [18.09045226 97.59392149]
Validation Recall Scores: [38.70967742 93.41414141]
Validation F1 Scores: [24.65753425 95.45829893]


Threshold: 0.9
Validation Accuracy: 93.6916
Validation Precision Scores: [14.43298969 96.8029138 ]
Validation Recall Scores: [15.05376344 96.64646465]
Validation F1 Scores: [14.73684211 96.72462596]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 320
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 65.8963, Val Loss: 45.0337
Epoch 10/50, Train Loss: 45.3449, Val Loss: 113.9213
Epoch 15/50, Train Loss: 40.8135, Val Loss: 68.8911
Epoch 20/50, Train Loss: 40.7620, Val Loss: 42.8764
Epoch 25/50, Train Loss: 49.0298, Val Loss: 47.2294
Early stopping triggered after 27 epochs because no improvements over 10 subsequent epochs!
Epoch 27/50, Train Loss: 40.5243, Val Loss: 43.1132
The function took 80.86 seconds to execute.
No description has been provided for this image
Training Accuracy: 76.7965
Training Precision Scores: [13.84672344 99.26014004]
Training Recall Scores: [86.97674419 76.35162602]
Training F1 Scores: [23.89013095 86.31167787]

Validation Accuracy: 76.1682
Validation Precision Scores: [11.44130758 99.15567282]
Validation Recall Scores: [82.79569892 75.91919192]
Validation F1 Scores: [20.10443864 85.99542334]

Threshold: 0.5
Validation Accuracy: 76.1682
Validation Precision Scores: [11.44130758 99.15567282]
Validation Recall Scores: [82.79569892 75.91919192]
Validation F1 Scores: [20.10443864 85.99542334]


Threshold: 0.6
Validation Accuracy: 83.9564
Validation Precision Scores: [15.09846827 98.86309806]
Validation Recall Scores: [74.19354839 84.32323232]
Validation F1 Scores: [25.09090909 91.01613607]


Threshold: 0.7
Validation Accuracy: 90.4984
Validation Precision Scores: [19.18367347 98.01980198]
Validation Recall Scores: [50.53763441 92.        ]
Validation F1 Scores: [27.81065089 94.91454773]


Threshold: 0.8
Validation Accuracy: 94.4704
Validation Precision Scores: [29.75206612 97.67061708]
Validation Recall Scores: [38.70967742 96.56565657]
Validation F1 Scores: [33.64485981 97.1149939 ]


Threshold: 0.9
Validation Accuracy: 95.9112
Validation Precision Scores: [40.90909091 97.36211031]
Validation Recall Scores: [29.03225806 98.42424242]
Validation F1 Scores: [33.96226415 97.89029536]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 384
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 45.9954, Val Loss: 61.7473
Epoch 10/50, Train Loss: 62.1685, Val Loss: 46.9586
Early stopping triggered after 11 epochs because no improvements over 10 subsequent epochs!
Epoch 11/50, Train Loss: 53.7007, Val Loss: 100.3963
The function took 36.98 seconds to execute.
No description has been provided for this image
Training Accuracy: 62.5998
Training Precision Scores: [ 9.30565497 99.34199704]
Training Recall Scores: [90.69767442 61.37195122]
Training F1 Scores: [16.87946332 75.87159997]

Validation Accuracy: 62.1106
Validation Precision Scores: [ 8.33333333 99.66931217]
Validation Recall Scores: [94.62365591 60.88888889]
Validation F1 Scores: [15.31766754 75.59568598]

Threshold: 0.5
Validation Accuracy: 62.1106
Validation Precision Scores: [ 8.33333333 99.66931217]
Validation Recall Scores: [94.62365591 60.88888889]
Validation F1 Scores: [15.31766754 75.59568598]


Threshold: 0.6
Validation Accuracy: 71.0280
Validation Precision Scores: [ 9.86436498 99.26010245]
Validation Recall Scores: [86.02150538 70.46464646]
Validation F1 Scores: [17.69911504 82.41965974]


Threshold: 0.7
Validation Accuracy: 82.5935
Validation Precision Scores: [13.87755102 98.79692012]
Validation Recall Scores: [73.11827957 82.94949495]
Validation F1 Scores: [23.32761578 90.18229739]


Threshold: 0.8
Validation Accuracy: 91.3551
Validation Precision Scores: [19.43127962 97.79380569]
Validation Recall Scores: [44.08602151 93.13131313]
Validation F1 Scores: [26.97368421 95.40562914]


Threshold: 0.9
Validation Accuracy: 95.5607
Validation Precision Scores: [28.57142857 96.86383486]
Validation Recall Scores: [15.05376344 98.58585859]
Validation F1 Scores: [19.71830986 97.71726071]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 64 and hidden dim 512
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 53.5404, Val Loss: 42.1515
Epoch 10/50, Train Loss: 57.1636, Val Loss: 56.8081
Early stopping triggered after 15 epochs because no improvements over 10 subsequent epochs!
Epoch 15/50, Train Loss: 53.8153, Val Loss: 48.0214
The function took 58.94 seconds to execute.
No description has been provided for this image
Training Accuracy: 79.2308
Training Precision Scores: [13.28158689 98.46560181]
Training Recall Scores: [71.62790698 79.56300813]
Training F1 Scores: [22.40814842 88.01079197]

Validation Accuracy: 79.1277
Validation Precision Scores: [11.6117851  98.69412356]
Validation Recall Scores: [72.04301075 79.39393939]
Validation F1 Scores: [20.         87.99820869]

Threshold: 0.5
Validation Accuracy: 79.1277
Validation Precision Scores: [11.6117851  98.69412356]
Validation Recall Scores: [72.04301075 79.39393939]
Validation F1 Scores: [20.         87.99820869]


Threshold: 0.6
Validation Accuracy: 83.2555
Validation Precision Scores: [12.96703297 98.39091339]
Validation Recall Scores: [63.44086022 84.        ]
Validation F1 Scores: [21.53284672 90.6277245 ]


Threshold: 0.7
Validation Accuracy: 91.6667
Validation Precision Scores: [18.97435897 97.64011799]
Validation Recall Scores: [39.78494624 93.61616162]
Validation F1 Scores: [25.69444444 95.58580858]


Threshold: 0.8
Validation Accuracy: 94.8598
Validation Precision Scores: [29.03225806 97.33333333]
Validation Recall Scores: [29.03225806 97.33333333]
Validation F1 Scores: [29.03225806 97.33333333]


Threshold: 0.9
Validation Accuracy: 95.8723
Validation Precision Scores: [30.3030303  96.72583826]
Validation Recall Scores: [10.75268817 99.07070707]
Validation F1 Scores: [15.87301587 97.88423154]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 128
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 35.4427, Val Loss: 39.1711
Epoch 10/50, Train Loss: 30.2086, Val Loss: 36.6473
Epoch 15/50, Train Loss: 30.8356, Val Loss: 42.9238
Epoch 20/50, Train Loss: 41.7856, Val Loss: 52.1033
Early stopping triggered after 21 epochs because no improvements over 10 subsequent epochs!
Epoch 21/50, Train Loss: 32.0611, Val Loss: 46.7448
The function took 52.01 seconds to execute.
No description has been provided for this image
Training Accuracy: 85.7644
Training Precision Scores: [19.61130742 98.86840877]
Training Recall Scores: [77.44186047 86.12804878]
Training F1 Scores: [31.29699248 92.0595264 ]

Validation Accuracy: 84.3069
Validation Precision Scores: [15.55555556 98.91406988]
Validation Recall Scores: [75.2688172  84.64646465]
Validation F1 Scores: [25.78268877 91.22577836]

Threshold: 0.5
Validation Accuracy: 84.3069
Validation Precision Scores: [15.55555556 98.91406988]
Validation Recall Scores: [75.2688172  84.64646465]
Validation F1 Scores: [25.78268877 91.22577836]


Threshold: 0.6
Validation Accuracy: 86.9548
Validation Precision Scores: [16.57458564 98.50407978]
Validation Recall Scores: [64.51612903 87.7979798 ]
Validation F1 Scores: [26.37362637 92.84340953]


Threshold: 0.7
Validation Accuracy: 89.4081
Validation Precision Scores: [18.14946619 98.16353301]
Validation Recall Scores: [54.83870968 90.70707071]
Validation F1 Scores: [27.27272727 94.28811424]


Threshold: 0.8
Validation Accuracy: 91.4330
Validation Precision Scores: [20.46511628 97.91755206]
Validation Recall Scores: [47.31182796 93.09090909]
Validation F1 Scores: [28.57142857 95.44324772]


Threshold: 0.9
Validation Accuracy: 94.7819
Validation Precision Scores: [30.09708738 97.48478702]
Validation Recall Scores: [33.33333333 97.09090909]
Validation F1 Scores: [31.63265306 97.28744939]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 256
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 44.7740, Val Loss: 48.9682
Epoch 10/50, Train Loss: 41.2241, Val Loss: 41.0986
Epoch 15/50, Train Loss: 41.6777, Val Loss: 38.3102
Early stopping triggered after 19 epochs because no improvements over 10 subsequent epochs!
Epoch 19/50, Train Loss: 42.4159, Val Loss: 42.3838
The function took 55.69 seconds to execute.
No description has been provided for this image
Training Accuracy: 77.1081
Training Precision Scores: [13.57603337 99.05672737]
Training Recall Scores: [83.25581395 76.83943089]
Training F1 Scores: [23.34528856 86.54495507]

Validation Accuracy: 75.8178
Validation Precision Scores: [11.62790698 99.30851064]
Validation Recall Scores: [86.02150538 75.43434343]
Validation F1 Scores: [20.4865557  85.74052813]

Threshold: 0.5
Validation Accuracy: 75.8178
Validation Precision Scores: [11.62790698 99.30851064]
Validation Recall Scores: [86.02150538 75.43434343]
Validation F1 Scores: [20.4865557  85.74052813]


Threshold: 0.6
Validation Accuracy: 80.7243
Validation Precision Scores: [13.32116788 99.00990099]
Validation Recall Scores: [78.49462366 80.80808081]
Validation F1 Scores: [22.77691108 88.98776418]


Threshold: 0.7
Validation Accuracy: 87.0717
Validation Precision Scores: [17.4386921  98.68241708]
Validation Recall Scores: [68.8172043  87.75757576]
Validation F1 Scores: [27.82608696 92.89991446]


Threshold: 0.8
Validation Accuracy: 89.7196
Validation Precision Scores: [17.97752809 98.04432855]
Validation Recall Scores: [51.61290323 91.15151515]
Validation F1 Scores: [26.66666667 94.47236181]


Threshold: 0.9
Validation Accuracy: 93.2243
Validation Precision Scores: [24.20382166 97.71878888]
Validation Recall Scores: [40.86021505 95.19191919]
Validation F1 Scores: [30.4        96.43880475]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 320
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 42.8486, Val Loss: 57.9186
Epoch 10/50, Train Loss: 47.2935, Val Loss: 47.7640
Epoch 15/50, Train Loss: 48.6684, Val Loss: 45.6595
Early stopping triggered after 19 epochs because no improvements over 10 subsequent epochs!
Epoch 19/50, Train Loss: 85.9424, Val Loss: 76.7440
The function took 62.29 seconds to execute.
No description has been provided for this image
Training Accuracy: 83.3204
Training Precision Scores: [17.35368957 98.92835641]
Training Recall Scores: [79.30232558 83.49593496]
Training F1 Scores: [28.47599165 90.55938275]

Validation Accuracy: 83.0607
Validation Precision Scores: [14.81481481 98.99135447]
Validation Recall Scores: [77.41935484 83.27272727]
Validation F1 Scores: [24.87046632 90.45424621]

Threshold: 0.5
Validation Accuracy: 83.0607
Validation Precision Scores: [14.81481481 98.99135447]
Validation Recall Scores: [77.41935484 83.27272727]
Validation F1 Scores: [24.87046632 90.45424621]


Threshold: 0.6
Validation Accuracy: 87.5389
Validation Precision Scores: [17.10144928 98.47053531]
Validation Recall Scores: [63.44086022 88.44444444]
Validation F1 Scores: [26.94063927 93.18859089]


Threshold: 0.7
Validation Accuracy: 90.3816
Validation Precision Scores: [18.69918699 97.97588286]
Validation Recall Scores: [49.46236559 91.91919192]
Validation F1 Scores: [27.13864307 94.85094851]


Threshold: 0.8
Validation Accuracy: 92.6402
Validation Precision Scores: [22.4137931 97.7443609]
Validation Recall Scores: [41.93548387 94.54545455]
Validation F1 Scores: [29.21348315 96.11829945]


Threshold: 0.9
Validation Accuracy: 94.3925
Validation Precision Scores: [25.71428571 97.32034105]
Validation Recall Scores: [29.03225806 96.84848485]
Validation F1 Scores: [27.27272727 97.08383961]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 384
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 62.2723, Val Loss: 46.9413
Epoch 10/50, Train Loss: 44.6124, Val Loss: 38.9098
Epoch 15/50, Train Loss: 51.3315, Val Loss: 51.0891
Early stopping triggered after 16 epochs because no improvements over 10 subsequent epochs!
Epoch 16/50, Train Loss: 52.7822, Val Loss: 40.0075
The function took 57.67 seconds to execute.
No description has been provided for this image
Training Accuracy: 72.5219
Training Precision Scores: [11.91082803 99.21458626]
Training Recall Scores: [86.97674419 71.8902439 ]
Training F1 Scores: [20.95238095 83.3706541 ]

Validation Accuracy: 71.7679
Validation Precision Scores: [ 9.89847716 99.15730337]
Validation Recall Scores: [83.87096774 71.31313131]
Validation F1 Scores: [17.70715096 82.96122209]

Threshold: 0.5
Validation Accuracy: 71.7679
Validation Precision Scores: [ 9.89847716 99.15730337]
Validation Recall Scores: [83.87096774 71.31313131]
Validation F1 Scores: [17.70715096 82.96122209]


Threshold: 0.6
Validation Accuracy: 82.1262
Validation Precision Scores: [12.65306122 98.50818094]
Validation Recall Scores: [66.66666667 82.70707071]
Validation F1 Scores: [21.26929674 89.9187349 ]


Threshold: 0.7
Validation Accuracy: 87.8505
Validation Precision Scores: [16.09907121 98.17371938]
Validation Recall Scores: [55.91397849 89.05050505]
Validation F1 Scores: [25.         93.38983051]


Threshold: 0.8
Validation Accuracy: 90.5374
Validation Precision Scores: [19.00826446 97.97936371]
Validation Recall Scores: [49.46236559 92.08080808]
Validation F1 Scores: [27.46268657 94.93855447]


Threshold: 0.9
Validation Accuracy: 93.4969
Validation Precision Scores: [24.30555556 97.60726073]
Validation Recall Scores: [37.6344086 95.5959596]
Validation F1 Scores: [29.53586498 96.59114105]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 128 and hidden dim 512
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 59.6383, Val Loss: 141.1408
Epoch 10/50, Train Loss: 67.5209, Val Loss: 148.3487
Early stopping triggered after 11 epochs because no improvements over 10 subsequent epochs!
Epoch 11/50, Train Loss: 112.2532, Val Loss: 67.8539
The function took 46.21 seconds to execute.
No description has been provided for this image
Training Accuracy: 81.9474
Training Precision Scores: [13.67346939 98.05054152]
Training Recall Scores: [62.3255814  82.80487805]
Training F1 Scores: [22.42677824 89.78512397]

Validation Accuracy: 82.5156
Validation Precision Scores: [12.60504202 98.42256214]
Validation Recall Scores: [64.51612903 83.19191919]
Validation F1 Scores: [21.08963093 90.16860083]

Threshold: 0.5
Validation Accuracy: 82.5156
Validation Precision Scores: [12.60504202 98.42256214]
Validation Recall Scores: [64.51612903 83.19191919]
Validation F1 Scores: [21.08963093 90.16860083]


Threshold: 0.6
Validation Accuracy: 89.6028
Validation Precision Scores: [17.29323308 97.95829713]
Validation Recall Scores: [49.46236559 91.11111111]
Validation F1 Scores: [25.62674095 94.41071802]


Threshold: 0.7
Validation Accuracy: 93.0296
Validation Precision Scores: [19.71830986 97.3206925 ]
Validation Recall Scores: [30.10752688 95.39393939]
Validation F1 Scores: [23.82978723 96.34768415]


Threshold: 0.8
Validation Accuracy: 96.2617
Validation Precision Scores: [44.82758621 96.84915321]
Validation Recall Scores: [13.97849462 99.35353535]
Validation F1 Scores: [21.31147541 98.08536099]


Threshold: 0.9
Validation Accuracy: 96.2617
Validation Precision Scores: [40.         96.59224442]
Validation Recall Scores: [ 6.4516129  99.63636364]
Validation F1 Scores: [11.11111111 98.09069212]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 256
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 45.7443, Val Loss: 37.2501
Epoch 10/50, Train Loss: 49.9239, Val Loss: 47.0525
Early stopping triggered after 14 epochs because no improvements over 10 subsequent epochs!
Epoch 14/50, Train Loss: 64.8983, Val Loss: 44.1815
The function took 49.37 seconds to execute.
No description has been provided for this image
Training Accuracy: 75.1412
Training Precision Scores: [13.25718242 99.36322991]
Training Recall Scores: [89.06976744 74.53252033]
Training F1 Scores: [23.07924074 85.17507694]

Validation Accuracy: 74.5717
Validation Precision Scores: [11.74863388 99.61873638]
Validation Recall Scores: [92.47311828 73.8989899 ]
Validation F1 Scores: [20.84848485 84.85270239]

Threshold: 0.5
Validation Accuracy: 74.5717
Validation Precision Scores: [11.74863388 99.61873638]
Validation Recall Scores: [92.47311828 73.8989899 ]
Validation F1 Scores: [20.84848485 84.85270239]


Threshold: 0.6
Validation Accuracy: 82.4766
Validation Precision Scores: [15.33980583 99.31807112]
Validation Recall Scores: [84.94623656 82.38383838]
Validation F1 Scores: [25.98684211 90.06183746]


Threshold: 0.7
Validation Accuracy: 89.5249
Validation Precision Scores: [21.05263158 98.71908127]
Validation Recall Scores: [68.8172043 90.3030303]
Validation F1 Scores: [32.2418136  94.32369698]


Threshold: 0.8
Validation Accuracy: 92.5234
Validation Precision Scores: [26.54028436 98.43020789]
Validation Recall Scores: [60.21505376 93.73737374]
Validation F1 Scores: [36.84210526 96.02649007]


Threshold: 0.9
Validation Accuracy: 93.9642
Validation Precision Scores: [29.05405405 97.9338843 ]
Validation Recall Scores: [46.23655914 95.75757576]
Validation F1 Scores: [35.6846473  96.83350358]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 320
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 45.8144, Val Loss: 147.2531
Epoch 10/50, Train Loss: 58.0587, Val Loss: 57.4010
Early stopping triggered after 11 epochs because no improvements over 10 subsequent epochs!
Epoch 11/50, Train Loss: 128.1200, Val Loss: 153.7509
The function took 42.92 seconds to execute.
No description has been provided for this image
Training Accuracy: 76.2999
Training Precision Scores: [12.9985229  98.96852684]
Training Recall Scores: [81.86046512 76.05691057]
Training F1 Scores: [22.43467177 86.01310194]

Validation Accuracy: 74.6495
Validation Precision Scores: [10.92436975 99.19093851]
Validation Recall Scores: [83.87096774 74.3030303 ]
Validation F1 Scores: [19.33085502 84.96188496]

Threshold: 0.5
Validation Accuracy: 74.6495
Validation Precision Scores: [10.92436975 99.19093851]
Validation Recall Scores: [83.87096774 74.3030303 ]
Validation F1 Scores: [19.33085502 84.96188496]


Threshold: 0.6
Validation Accuracy: 81.7368
Validation Precision Scores: [12.8458498  98.64209505]
Validation Recall Scores: [69.89247312 82.18181818]
Validation F1 Scores: [21.70283806 89.66277276]


Threshold: 0.7
Validation Accuracy: 87.8505
Validation Precision Scores: [13.62126246 97.70621967]
Validation Recall Scores: [44.08602151 89.49494949]
Validation F1 Scores: [20.81218274 93.42049768]


Threshold: 0.8
Validation Accuracy: 93.1075
Validation Precision Scores: [18.18181818 97.16748768]
Validation Recall Scores: [25.80645161 95.63636364]
Validation F1 Scores: [21.33333333 96.39584606]


Threshold: 0.9
Validation Accuracy: 95.7944
Validation Precision Scores: [30.76923077 96.79715302]
Validation Recall Scores: [12.90322581 98.90909091]
Validation F1 Scores: [18.18181818 97.84172662]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 384
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 50.6457, Val Loss: 54.0096
Epoch 10/50, Train Loss: 57.9158, Val Loss: 63.1784
Early stopping triggered after 11 epochs because no improvements over 10 subsequent epochs!
Epoch 11/50, Train Loss: 55.0177, Val Loss: 81.5015
The function took 46.51 seconds to execute.
No description has been provided for this image
Training Accuracy: 78.2181
Training Precision Scores: [13.67108967 98.84363356]
Training Recall Scores: [79.06976744 78.18089431]
Training F1 Scores: [23.31162153 87.30636101]

Validation Accuracy: 76.3629
Validation Precision Scores: [11.29518072 99.05462185]
Validation Recall Scores: [80.64516129 76.2020202 ]
Validation F1 Scores: [19.81505945 86.13838776]

Threshold: 0.5
Validation Accuracy: 76.3629
Validation Precision Scores: [11.29518072 99.05462185]
Validation Recall Scores: [80.64516129 76.2020202 ]
Validation F1 Scores: [19.81505945 86.13838776]


Threshold: 0.6
Validation Accuracy: 86.9548
Validation Precision Scores: [16.01123596 98.37251356]
Validation Recall Scores: [61.29032258 87.91919192]
Validation F1 Scores: [25.38975501 92.85257094]


Threshold: 0.7
Validation Accuracy: 92.4844
Validation Precision Scores: [24.22680412 98.06234204]
Validation Recall Scores: [50.53763441 94.06060606]
Validation F1 Scores: [32.75261324 96.0197979 ]


Threshold: 0.8
Validation Accuracy: 94.5483
Validation Precision Scores: [31.78294574 97.86797868]
Validation Recall Scores: [44.08602151 96.44444444]
Validation F1 Scores: [36.93693694 97.15099715]


Threshold: 0.9
Validation Accuracy: 96.3396
Validation Precision Scores: [49.09090909 97.37365698]
Validation Recall Scores: [29.03225806 98.86868687]
Validation F1 Scores: [36.48648649 98.11547715]


-------------------------------------------------------------------------
-------------------------------------------------------------------------
-------------------------------------------------------------------------
Trying with embedding size 256 and hidden dim 512
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 88.0098, Val Loss: 67.3749
Epoch 10/50, Train Loss: 44.6553, Val Loss: 51.4087
Epoch 15/50, Train Loss: 60.4612, Val Loss: 82.3411
Epoch 20/50, Train Loss: 54.3925, Val Loss: 223.0944
Early stopping triggered after 23 epochs because no improvements over 10 subsequent epochs!
Epoch 23/50, Train Loss: 72.2414, Val Loss: 53.0238
The function took 113.05 seconds to execute.
No description has been provided for this image
Training Accuracy: 92.5998
Training Precision Scores: [27.7027027  97.63903463]
Training Recall Scores: [47.6744186  94.56300813]
Training F1 Scores: [35.04273504 96.07640681]

Validation Accuracy: 93.0296
Validation Precision Scores: [23.7804878  97.75374376]
Validation Recall Scores: [41.93548387 94.94949495]
Validation F1 Scores: [30.35019455 96.33121541]

Threshold: 0.5
Validation Accuracy: 93.0296
Validation Precision Scores: [23.7804878  97.75374376]
Validation Recall Scores: [41.93548387 94.94949495]
Validation F1 Scores: [30.35019455 96.33121541]


Threshold: 0.6
Validation Accuracy: 94.2368
Validation Precision Scores: [28.34645669 97.66489144]
Validation Recall Scores: [38.70967742 96.32323232]
Validation F1 Scores: [32.72727273 96.98942229]


Threshold: 0.7
Validation Accuracy: 95.1713
Validation Precision Scores: [32.96703297 97.45660073]
Validation Recall Scores: [32.25806452 97.53535354]
Validation F1 Scores: [32.60869565 97.49596123]


Threshold: 0.8
Validation Accuracy: 95.6776
Validation Precision Scores: [37.14285714 97.31785428]
Validation Recall Scores: [27.95698925 98.22222222]
Validation F1 Scores: [31.90184049 97.76794691]


Threshold: 0.9
Validation Accuracy: 96.0280
Validation Precision Scores: [40.         97.02734839]
Validation Recall Scores: [19.35483871 98.90909091]
Validation F1 Scores: [26.08695652 97.95918367]


In [11]:
#16 and hidden dim 16  threhsol 0.8

embedding_size=16
hidden_dim=16
threshold = 0.8

# Create embedding layer with the given vocabulary size and embedding size
vocabulary_size = len(vectorizer.vocabulary_) + 1  # Add 1 for out-of-vocabulary
embeddings = nn.Embedding(vocabulary_size, embedding_size)
X_train_graph = GraphDataset(X_train, y_train, embeddings)
X_val_graph = GraphDataset(X_val, y_val, embeddings)
X_test_graph = GraphDataset(X_test, y_test, embeddings)

# Create DataLoader for batching
# NOTICE: This data loader is from torch_geometric.loader!! It is not from torch.utils.data
batch_size = 64
train_loader = torch_geometric.loader.DataLoader(X_train_graph, batch_size=batch_size, shuffle=True)
val_loader = torch_geometric.loader.DataLoader(X_val_graph, batch_size=batch_size, shuffle=False)
test_loader = torch_geometric.loader.DataLoader(X_test_graph, batch_size=batch_size, shuffle=False)

class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train_original), y=y_train_original)  # Computing class weights
class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
num_epochs = 50
model = GAT(input_dim=embedding_size, hidden_dim=hidden_dim, output_dim=2).to(device)
optimizer = optim.AdamW(model.parameters(), lr=0.01, weight_decay=0.01)
training_loop(model, train_loader, val_loader, X_train_graph, X_val_graph, device, optimizer, num_epochs, criterion)

test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model_with_threshold(model,val_loader,device,threshold=threshold)

stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 34.6077, Val Loss: 33.0638
Epoch 10/50, Train Loss: 29.7320, Val Loss: 27.8456
Epoch 15/50, Train Loss: 26.9065, Val Loss: 25.7897
Epoch 20/50, Train Loss: 23.3077, Val Loss: 26.8553
Epoch 25/50, Train Loss: 24.6751, Val Loss: 27.1542
Epoch 30/50, Train Loss: 21.1759, Val Loss: 26.9506
Early stopping triggered after 32 epochs because no improvements over 10 subsequent epochs!
Epoch 32/50, Train Loss: 20.9676, Val Loss: 27.2604
The function took 50.99 seconds to execute.
No description has been provided for this image
Test Accuracy: 94.4315
Test Precision Scores: [35.63218391 98.70509607]
Test Recall Scores: [66.66666667 95.47474747]
Test F1 Scores: [46.44194757 97.06305196]

Out[11]:
<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7d3f7e3ea7d0>
No description has been provided for this image
No description has been provided for this image

Simple GCN on CPU¶

In [14]:
# 16 and hidden dim 128 threshold 0.9

device = 'cpu'
embedding_size=16
hidden_dim=128
threshold = 0.9 

# Create embedding layer with the given vocabulary size and embedding size
vocabulary_size = len(vectorizer.vocabulary_) + 1  # Add 1 for out-of-vocabulary
embeddings = nn.Embedding(vocabulary_size, embedding_size)
X_train_graph = GraphDataset(X_train, y_train, embeddings)
X_val_graph = GraphDataset(X_val, y_val, embeddings)
X_test_graph = GraphDataset(X_test, y_test, embeddings)

# Create DataLoader for batching
# NOTICE: This data loader is from torch_geometric.loader!! It is not from torch.utils.data
batch_size = 64
train_loader = torch_geometric.loader.DataLoader(X_train_graph, batch_size=batch_size, shuffle=True)
val_loader = torch_geometric.loader.DataLoader(X_val_graph, batch_size=batch_size, shuffle=False)
test_loader = torch_geometric.loader.DataLoader(X_test_graph, batch_size=batch_size, shuffle=False)

class_weight = compute_class_weight(class_weight='balanced', classes=np.unique(y_train_original), y=y_train_original)  # Computing class weights
class_weight_tensor = torch.tensor(class_weight, dtype=torch.float32).to(device)  # Converting to PyTorch tensor
criterion = nn.CrossEntropyLoss(weight=class_weight_tensor)
num_epochs = 50
model = GCN(input_dim=embedding_size, hidden_dim=hidden_dim, output_dim=2).to(device)
optimizer = optim.AdamW(model.parameters(), lr=0.01, weight_decay=0.01)
training_loop(model, train_loader, val_loader, X_train_graph, X_val_graph, device, optimizer, num_epochs, criterion)

train_accuracy, train_precisionscores, train_recallscores, train_f1scores, _, _ = testing_model(model,train_loader,device)
val_accuracy, val_precisionscores, val_recallscores, val_f1scores, _, _ = testing_model(model,val_loader,device)
test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model(model,test_loader,device)

stats(train_accuracy, train_precisionscores, train_recallscores, train_f1scores, 'Training')
stats(val_accuracy, val_precisionscores, val_recallscores, val_f1scores, 'Validation')
stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, 'Test')

# Test with threshold
print(f"Threshold: {threshold}")
test_accuracy, test_precisionscores, test_recallscores, test_f1scores, all_labels_test, all_predictions_test = testing_model_with_threshold(model, test_loader, device, threshold=threshold)
stats(test_accuracy, test_precisionscores, test_recallscores, test_f1scores, f'Test {threshold}')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, cmap='Blues')
ConfusionMatrixDisplay.from_predictions(all_labels_test, all_predictions_test, normalize='true', cmap='Blues')
print()
  0%|          | 0/50 [00:00<?, ?it/s]
Epoch 5/50, Train Loss: 36.8246, Val Loss: 35.8298
Epoch 10/50, Train Loss: 33.8961, Val Loss: 32.5482
Epoch 15/50, Train Loss: 31.0544, Val Loss: 34.1158
Epoch 20/50, Train Loss: 29.4902, Val Loss: 29.2766
Epoch 25/50, Train Loss: 27.9879, Val Loss: 30.2824
Epoch 30/50, Train Loss: 27.1075, Val Loss: 26.0054
Epoch 35/50, Train Loss: 26.2199, Val Loss: 27.2533
Epoch 40/50, Train Loss: 25.4207, Val Loss: 25.0337
Epoch 45/50, Train Loss: 25.4130, Val Loss: 26.8479
Early stopping triggered after 50 epochs because no improvements over 10 subsequent epochs!
Epoch 50/50, Train Loss: 24.1975, Val Loss: 28.1010
The function took 204.92 seconds to execute.
No description has been provided for this image
Training Accuracy: 87.8968
Training Precision Scores: [24.41787288 99.51618477]
Training Recall Scores: [90.23255814 87.79471545]
Training F1 Scores: [38.43486875 93.28869931]

Validation Accuracy: 87.1106
Validation Precision Scores: [20.10050251 99.40092166]
Validation Recall Scores: [86.02150538 87.15151515]
Validation F1 Scores: [32.58655804 92.87405813]

Test Accuracy: 87.7398
Test Precision Scores: [19.14691943 99.23736892]
Test Recall Scores: [80.8        87.98422313]
Test F1 Scores: [30.95785441 93.27260509]

Threshold: 0.9
Test 0.9 Accuracy: 97.1425
Test 0.9 Precision Scores: [64.49275362 97.76729996]
Test 0.9 Recall Scores: [35.6        99.30976194]
Test 0.9 F1 Scores: [45.87628866 98.53249476]


No description has been provided for this image
No description has been provided for this image